Include total difficulty in CHTs and hide implementation details from…

… consumers (#4428)

* CHT builder and prover

* use CHT abstraction in provider

* hide CHT internals from header chain

* fix itertools conflict by updating all to 0.5

* cht proof checker, use it in on_demand

ethcore/light/Cargo.toml

@@ -22,6 +22,7 @@ time = "0.1"
 smallvec = "0.3.1"
 futures = "0.1"
 rand = "0.3"
+itertools = "0.5"
 
 [features]
 default = []

ethcore/light/src/cht.rs

@@ -12,10 +12,154 @@
 // GNU General Public License for more details.
 
 //! Canonical hash trie definitions and helper functions.
+//!
+//! Each CHT is a trie mapping block numbers to canonical hashes and total difficulty.
+//! One is generated for every `SIZE` blocks, allowing us to discard those blocks in
+//! favor the the trie root. When the "ancient" blocks need to be accessed, we simply
+//! request an inclusion proof of a specific block number against the trie with the
+//! root has. A correct proof implies that the claimed block is identical to the one
+//! we discarded.
+
+use ethcore::ids::BlockId;
+use util::{Bytes, H256, U256, HashDB, MemoryDB};
+use util::trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder};
+use rlp::{Stream, RlpStream, UntrustedRlp, View};
+
+// encode a key.
+macro_rules! key {
+	($num: expr) => { ::rlp::encode(&$num) }
+}
+
+macro_rules! val {
+	($hash: expr, $td: expr) => {{
+		let mut stream = RlpStream::new_list(2);
+		stream.append(&$hash).append(&$td);
+		stream.drain()
+	}}
+}
 
 /// The size of each CHT.
 pub const SIZE: u64 = 2048;
 
+/// A canonical hash trie. This is generic over any database it can query.
+/// See module docs for more details.
+#[derive(Debug, Clone)]
+pub struct CHT<DB: HashDB> {
+	db: DB,
+	root: H256, // the root of this CHT.
+	number: u64,
+}
+
+impl<DB: HashDB> CHT<DB> {
+	/// Query the root of the CHT.
+	pub fn root(&self) -> H256 { self.root }
+
+	/// Query the number of the CHT.
+	pub fn number(&self) -> u64 { self.number }
+
+	/// Generate an inclusion proof for the entry at a specific block.
+	/// Nodes before level `from_level` will be omitted.
+	/// Returns an error on an incomplete trie, and `Ok(None)` on an unprovable request.
+	pub fn prove(&self, num: u64, from_level: u32) -> trie::Result<Option<Vec<Bytes>>> {
+		if block_to_cht_number(num) != Some(self.number) { return Ok(None) }
+
+		let mut recorder = Recorder::with_depth(from_level);
+		let t = TrieDB::new(&self.db, &self.root)?;
+		t.get_with(&key!(num), &mut recorder)?;
+
+		Ok(Some(recorder.drain().into_iter().map(|x| x.data).collect()))
+	}
+}
+
+/// Block information necessary to build a CHT.
+pub struct BlockInfo {
+	/// The block's hash.
+	pub hash: H256,
+	/// The block's parent's hash.
+	pub parent_hash: H256,
+	/// The block's total difficulty.
+	pub total_difficulty: U256,
+}
+
+/// Build an in-memory CHT from a closure which provides necessary information
+/// about blocks. If the fetcher ever fails to provide the info, the CHT
+/// will not be generated.
+pub fn build<F>(cht_num: u64, mut fetcher: F) ->  Option<CHT<MemoryDB>>
+	where F: FnMut(BlockId) -> Option<BlockInfo>
+{
+	let mut db = MemoryDB::new();
+
+	// start from the last block by number and work backwards.
+	let last_num = start_number(cht_num + 1) - 1;
+	let mut id = BlockId::Number(last_num);
+
+	let mut root = H256::default();
+
+	{
+		let mut t = TrieDBMut::new(&mut db, &mut root);
+		for blk_num in (0..SIZE).map(|n| last_num - n) {
+			let info = match fetcher(id) {
+				Some(info) => info,
+				None => return None,
+			};
+
+			id = BlockId::Hash(info.parent_hash);
+			t.insert(&key!(blk_num), &val!(info.hash, info.total_difficulty))
+				.expect("fresh in-memory database is infallible; qed");
+		}
+	}
+
+	Some(CHT {
+		db: db,
+		root: root,
+		number: cht_num,
+	})
+}
+
+/// Compute a CHT root from an iterator of (hash, td) pairs. Fails if shorter than
+/// SIZE items. The items are assumed to proceed sequentially from `start_number(cht_num)`.
+/// Discards the trie's nodes.
+pub fn compute_root<I>(cht_num: u64, iterable: I) -> Option<H256>
+	where I: IntoIterator<Item=(H256, U256)>
+{
+	let mut v = Vec::with_capacity(SIZE as usize);
+	let start_num = start_number(cht_num) as usize;
+
+	for (i, (h, td)) in iterable.into_iter().take(SIZE as usize).enumerate() {
+		v.push((key!(i + start_num).to_vec(), val!(h, td).to_vec()))
+	}
+
+	if v.len() == SIZE as usize {
+		Some(::util::triehash::trie_root(v))
+	} else {
+		None
+	}
+}
+
+/// Check a proof for a CHT.
+/// Given a set of a trie nodes, a number to query, and a trie root,
+/// verify the given trie branch and extract the canonical hash and total difficulty.
+// TODO: better support for partially-checked queries.
+pub fn check_proof(proof: &[Bytes], num: u64, root: H256) -> Option<(H256, U256)> {
+	let mut db = MemoryDB::new();
+
+	for node in proof { db.insert(&node[..]); }
+	let res = match TrieDB::new(&db, &root) {
+		Err(_) => return None,
+		Ok(trie) => trie.get_with(&key!(num), |val: &[u8]| {
+			let rlp = UntrustedRlp::new(val);
+			rlp.val_at::<H256>(0)
+				.and_then(|h| rlp.val_at::<U256>(1).map(|td| (h, td)))
+				.ok()
+		})
+	};
+
+	match res {
+		Ok(Some(Some((hash, td)))) => Some((hash, td)),
+		_ => None,
+	}
+}
+
 /// Convert a block number to a CHT number.
 /// Returns `None` for `block_num` == 0, `Some` otherwise.
 pub fn block_to_cht_number(block_num: u64) -> Option<u64> {
@@ -37,6 +181,12 @@ pub fn start_number(cht_num: u64) -> u64 {
 
 #[cfg(test)]
 mod tests {
+	#[test]
+	fn size_is_lt_usize() {
+		// to ensure safe casting on the target platform.
+		assert!(::cht::SIZE < usize::max_value() as u64)
+	}
+
 	#[test]
 	fn block_to_cht_number() {
 		assert!(::cht::block_to_cht_number(0).is_none());

ethcore/light/src/client/header_chain.rs

@@ -173,26 +173,34 @@ impl HeaderChain {
 			// produce next CHT root if it's time.
 			let earliest_era = *candidates.keys().next().expect("at least one era just created; qed");
 			if earliest_era + HISTORY + cht::SIZE <= number {
-				let mut values = Vec::with_capacity(cht::SIZE as usize);
-				{
-					let mut headers = self.headers.write();
-					for i in (0..cht::SIZE).map(|x| x + earliest_era) {
+				let cht_num = cht::block_to_cht_number(earliest_era)
+					.expect("fails only for number == 0; genesis never imported; qed");
+				debug_assert_eq!(cht_num as usize, self.cht_roots.lock().len());
+
+				let mut headers = self.headers.write();
+
+				let cht_root = {
+					let mut i = earliest_era;
+
+					// iterable function which removes the candidates as it goes
+					// along. this will only be called until the CHT is complete.
+					let iter = || {
 						let era_entry = candidates.remove(&i)
 							.expect("all eras are sequential with no gaps; qed");
+						i += 1;
 
 						for ancient in &era_entry.candidates {
 							headers.remove(&ancient.hash);
 						}
 
-						values.push((
-							::rlp::encode(&i).to_vec(),
-							::rlp::encode(&era_entry.canonical_hash).to_vec(),
-						));
-					}
-				}
+						let canon = &era_entry.candidates[0];
+						(canon.hash, canon.total_difficulty)
+					};
+					cht::compute_root(cht_num, ::itertools::repeat_call(iter))
+						.expect("fails only when too few items; this is checked; qed")
+				};
 
-				let cht_root = ::util::triehash::trie_root(values);
-				debug!(target: "chain", "Produced CHT {} root: {:?}", (earliest_era - 1) % cht::SIZE, cht_root);
+				debug!(target: "chain", "Produced CHT {} root: {:?}", cht_num, cht_root);
 
 				self.cht_roots.lock().push(cht_root);
 			}

ethcore/light/src/lib.rs

@@ -68,6 +68,7 @@ extern crate smallvec;
 extern crate time;
 extern crate futures;
 extern crate rand;
+extern crate itertools;
 
 #[cfg(feature = "ipc")]
 extern crate ethcore_ipc as ipc;

ethcore/light/src/on_demand/mod.rs

@@ -29,7 +29,7 @@ use futures::sync::oneshot;
 use network::PeerId;
 
 use net::{Handler, Status, Capabilities, Announcement, EventContext, BasicContext, ReqId};
-use util::{Bytes, RwLock};
+use util::{Bytes, RwLock, U256};
 use types::les_request::{self as les_request, Request as LesRequest};
 
 pub mod request;
@@ -79,7 +79,7 @@ struct Peer {
 
 // Attempted request info and sender to put received value.
 enum Pending {
-	HeaderByNumber(request::HeaderByNumber, Sender<encoded::Header>), // num + CHT root
+	HeaderByNumber(request::HeaderByNumber, Sender<(encoded::Header, U256)>), // num + CHT root
 	HeaderByHash(request::HeaderByHash, Sender<encoded::Header>),
 	Block(request::Body, Sender<encoded::Block>),
 	BlockReceipts(request::BlockReceipts, Sender<Vec<Receipt>>),
@@ -105,14 +105,15 @@ impl Default for OnDemand {
 
 impl OnDemand {
 	/// Request a header by block number and CHT root hash.
-	pub fn header_by_number(&self, ctx: &BasicContext, req: request::HeaderByNumber) -> Response<encoded::Header> {
+	/// Returns the header and the total difficulty.
+	pub fn header_by_number(&self, ctx: &BasicContext, req: request::HeaderByNumber) -> Response<(encoded::Header, U256)> {
 		let (sender, receiver) = oneshot::channel();
 		self.dispatch_header_by_number(ctx, req, sender);
 		Response(receiver)
 	}
 
 	// dispatch the request, completing the request if no peers available.
-	fn dispatch_header_by_number(&self, ctx: &BasicContext, req: request::HeaderByNumber, sender: Sender<encoded::Header>) {
+	fn dispatch_header_by_number(&self, ctx: &BasicContext, req: request::HeaderByNumber, sender: Sender<(encoded::Header, U256)>) {
 		let num = req.num;
 		let cht_num = match ::cht::block_to_cht_number(req.num) {
 			Some(cht_num) => cht_num,