Add ChainNotifier and define ChainListener trait

Add an interface for being notified of block connected and disconnected
events, along with a notifier for generating such events. Used while
polling block sources for a new tip in order to feed these events into
ChannelManager and ChainMonitor.

Author: jkczyz

lightning-block-sync/src/lib.rs

@@ -31,6 +31,8 @@ mod test_utils;
 #[cfg(any(feature = "rest-client", feature = "rpc-client"))]
 mod utils;
 
+use crate::poll::{Poll, ValidatedBlockHeader};
+
 use bitcoin::blockdata::block::{Block, BlockHeader};
 use bitcoin::hash_types::BlockHash;
 use bitcoin::util::uint::Uint256;
@@ -130,3 +132,336 @@ pub struct BlockHeaderData {
 	/// of equivalent weight.
 	pub chainwork: Uint256,
 }
+
+/// Adaptor used for notifying when blocks have been connected or disconnected from the chain.
+///
+/// Used when needing to replay chain data upon startup or as new chain events occur.
+pub trait ChainListener {
+	/// Notifies the listener that a block was added at the given height.
+	fn block_connected(&mut self, block: &Block, height: u32);
+
+	/// Notifies the listener that a block was removed at the given height.
+	fn block_disconnected(&mut self, header: &BlockHeader, height: u32);
+}
+
+/// The `Cache` trait defines behavior for managing a block header cache, where block headers are
+/// keyed by block hash.
+///
+/// Used by [`ChainNotifier`] to store headers along the best chain. Implementations may define
+/// their own cache eviction policy.
+///
+/// [`ChainNotifier`]: struct.ChainNotifier.html
+pub trait Cache {
+	/// Retrieves the block header keyed by the given block hash.
+	fn get(&self, block_hash: &BlockHash) -> Option<&ValidatedBlockHeader>;
+
+	/// Inserts a block header keyed by the given block hash.
+	fn insert(&mut self, block_hash: BlockHash, block_header: ValidatedBlockHeader);
+
+	/// Removes the block header keyed by the given block hash.
+	fn remove(&mut self, block_hash: &BlockHash) -> Option<ValidatedBlockHeader>;
+}
+
+/// Unbounded cache of block headers keyed by block hash.
+pub type UnboundedCache = std::collections::HashMap<BlockHash, ValidatedBlockHeader>;
+
+impl Cache for UnboundedCache {
+	fn get(&self, block_hash: &BlockHash) -> Option<&ValidatedBlockHeader> {
+		self.get(block_hash)
+	}
+
+	fn insert(&mut self, block_hash: BlockHash, block_header: ValidatedBlockHeader) {
+		self.insert(block_hash, block_header);
+	}
+
+	fn remove(&mut self, block_hash: &BlockHash) -> Option<ValidatedBlockHeader> {
+		self.remove(block_hash)
+	}
+}
+
+/// Notifies [listeners] of blocks that have been connected or disconnected from the chain.
+///
+/// [listeners]: trait.ChainListener.html
+struct ChainNotifier<C: Cache> {
+	/// Cache for looking up headers before fetching from a block source.
+	header_cache: C,
+}
+
+/// Steps outlining changes needed to be made to the chain in order to transform it from having one
+/// chain tip to another.
+enum ForkStep {
+	ForkPoint(ValidatedBlockHeader),
+	DisconnectBlock(ValidatedBlockHeader),
+	ConnectBlock(ValidatedBlockHeader),
+}
+
+impl<C: Cache> ChainNotifier<C> {
+	/// Finds the fork point between `new_header` and `old_header`, disconnecting blocks from
+	/// `old_header` to get to that point and then connecting blocks until `new_header`.
+	///
+	/// Validates headers along the transition path, but doesn't fetch blocks until the chain is
+	/// disconnected to the fork point. Thus, this may return an `Err` that includes where the tip
+	/// ended up which may not be `new_header`. Note that iff the returned `Err` contains `Some`
+	/// header then the transition from `old_header` to `new_header` is valid.
+	async fn sync_listener<L: ChainListener, P: Poll>(
+		&mut self,
+		new_header: ValidatedBlockHeader,
+		old_header: &ValidatedBlockHeader,
+		chain_poller: &mut P,
+		chain_listener: &mut L,
+	) -> Result<(), (BlockSourceError, Option<ValidatedBlockHeader>)> {
+		let mut events = self.find_fork(new_header, old_header, chain_poller).await.map_err(|e| (e, None))?;
+
+		let mut last_disconnect_tip = None;
+		let mut new_tip = None;
+		for event in events.iter() {
+			match &event {
+				&ForkStep::DisconnectBlock(ref header) => {
+					println!("Disconnecting block {}", header.block_hash);
+					if let Some(cached_header) = self.header_cache.remove(&header.block_hash) {
+						assert_eq!(cached_header, *header);
+					}
+					chain_listener.block_disconnected(&header.header, header.height);
+					last_disconnect_tip = Some(header.header.prev_blockhash);
+				},
+				&ForkStep::ForkPoint(ref header) => {
+					new_tip = Some(*header);
+				},
+				_ => {},
+			}
+		}
+
+		// If blocks were disconnected, new blocks will connect starting from the fork point.
+		// Otherwise, there was no fork, so new blocks connect starting from the old tip.
+		assert_eq!(last_disconnect_tip.is_some(), new_tip.is_some());
+		if let &Some(ref tip_header) = &new_tip {
+			debug_assert_eq!(tip_header.header.block_hash(), *last_disconnect_tip.as_ref().unwrap());
+		} else {
+			new_tip = Some(*old_header);
+		}
+
+		for event in events.drain(..).rev() {
+			if let ForkStep::ConnectBlock(header) = event {
+				let block = chain_poller
+					.fetch_block(&header).await
+					.or_else(|e| Err((e, new_tip)))?;
+				debug_assert_eq!(block.block_hash, header.block_hash);
+
+				println!("Connecting block {}", header.block_hash);
+				self.header_cache.insert(header.block_hash, header);
+				chain_listener.block_connected(&block, header.height);
+				new_tip = Some(header);
+			}
+		}
+		Ok(())
+	}
+
+	/// Walks backwards from `current_header` and `prev_header`, finding the common ancestor.
+	/// Returns the steps needed to produce the chain with `current_header` as its tip from the
+	/// chain with `prev_header` as its tip. There is no ordering guarantee between different
+	/// `ForkStep` types, but `DisconnectBlock` and `ConnectBlock` are each returned in
+	/// height-descending order.
+	async fn find_fork<P: Poll>(
+		&self,
+		current_header: ValidatedBlockHeader,
+		prev_header: &ValidatedBlockHeader,
+		chain_poller: &mut P,
+	) -> BlockSourceResult<Vec<ForkStep>> {
+		let mut steps = Vec::new();
+		let mut current = current_header;
+		let mut previous = *prev_header;
+		loop {
+			// Found the parent block.
+			if current.height == previous.height + 1 &&
+					current.header.prev_blockhash == previous.block_hash {
+				steps.push(ForkStep::ConnectBlock(current));
+				break;
+			}
+
+			// Found a chain fork.
+			if current.header.prev_blockhash == previous.header.prev_blockhash {
+				let fork_point = self.look_up_previous_header(chain_poller, &previous).await?;
+				steps.push(ForkStep::DisconnectBlock(previous));
+				steps.push(ForkStep::ConnectBlock(current));
+				steps.push(ForkStep::ForkPoint(fork_point));
+				break;
+			}
+
+			// Walk back the chain, finding blocks needed to connect and disconnect. Only walk back
+			// the header with the greater height, or both if equal heights.
+			let current_height = current.height;
+			let previous_height = previous.height;
+			if current_height <= previous_height {
+				steps.push(ForkStep::DisconnectBlock(previous));
+				previous = self.look_up_previous_header(chain_poller, &previous).await?;
+			}
+			if current_height >= previous_height {
+				steps.push(ForkStep::ConnectBlock(current));
+				current = self.look_up_previous_header(chain_poller, &current).await?;
+			}
+		}
+
+		Ok(steps)
+	}
+
+	/// Returns the previous header for the given header, either by looking it up in the cache or
+	/// fetching it if not found.
+	async fn look_up_previous_header<P: Poll>(
+		&self,
+		chain_poller: &mut P,
+		header: &ValidatedBlockHeader,
+	) -> BlockSourceResult<ValidatedBlockHeader> {
+		match self.header_cache.get(&header.header.prev_blockhash) {
+			Some(prev_header) => Ok(*prev_header),
+			None => chain_poller.look_up_previous_header(header).await,
+		}
+	}
+}
+
+#[cfg(test)]
+mod chain_notifier_tests {
+	use crate::test_utils::{Blockchain, MockChainListener};
+	use super::*;
+
+	use bitcoin::network::constants::Network;
+
+	#[tokio::test]
+	async fn sync_from_same_chain() {
+		let mut chain = Blockchain::default().with_height(3);
+
+		let new_tip = chain.tip();
+		let old_tip = chain.at_height(1);
+		let mut listener = MockChainListener::new()
+			.expect_block_connected(*chain.at_height(2))
+			.expect_block_connected(*new_tip);
+		let mut notifier = ChainNotifier { header_cache: chain.header_cache(0..=1) };
+		let mut poller = poll::ChainPoller::new(&mut chain as &mut dyn BlockSource, Network::Testnet);
+		match notifier.sync_listener(new_tip, &old_tip, &mut poller, &mut listener).await {
+			Err((e, _)) => panic!("Unexpected error: {:?}", e),
+			Ok(_) => {},
+		}
+	}
+
+	#[tokio::test]
+	async fn sync_from_different_chains() {
+		let mut test_chain = Blockchain::with_network(Network::Testnet).with_height(1);
+		let main_chain = Blockchain::with_network(Network::Bitcoin).with_height(1);
+
+		let new_tip = test_chain.tip();
+		let old_tip = main_chain.tip();
+		let mut listener = MockChainListener::new();
+		let mut notifier = ChainNotifier { header_cache: main_chain.header_cache(0..=1) };
+		let mut poller = poll::ChainPoller::new(&mut test_chain as &mut dyn BlockSource, Network::Testnet);
+		match notifier.sync_listener(new_tip, &old_tip, &mut poller, &mut listener).await {
+			Err((e, _)) => {
+				assert_eq!(e.kind(), BlockSourceErrorKind::Persistent);
+				assert_eq!(e.into_inner().as_ref().to_string(), "genesis block reached");
+			},
+			Ok(_) => panic!("Expected error"),
+		}
+	}
+
+	#[tokio::test]
+	async fn sync_from_equal_length_fork() {
+		let main_chain = Blockchain::default().with_height(2);
+		let mut fork_chain = main_chain.fork_at_height(1);
+
+		let new_tip = fork_chain.tip();
+		let old_tip = main_chain.tip();
+		let mut listener = MockChainListener::new()
+			.expect_block_disconnected(*old_tip)
+			.expect_block_connected(*new_tip);
+		let mut notifier = ChainNotifier { header_cache: main_chain.header_cache(0..=2) };
+		let mut poller = poll::ChainPoller::new(&mut fork_chain as &mut dyn BlockSource, Network::Testnet);
+		match notifier.sync_listener(new_tip, &old_tip, &mut poller, &mut listener).await {
+			Err((e, _)) => panic!("Unexpected error: {:?}", e),
+			Ok(_) => {},
+		}
+	}
+
+	#[tokio::test]
+	async fn sync_from_shorter_fork() {
+		let main_chain = Blockchain::default().with_height(3);
+		let mut fork_chain = main_chain.fork_at_height(1);
+		fork_chain.disconnect_tip();
+
+		let new_tip = fork_chain.tip();
+		let old_tip = main_chain.tip();
+		let mut listener = MockChainListener::new()
+			.expect_block_disconnected(*old_tip)
+			.expect_block_disconnected(*main_chain.at_height(2))
+			.expect_block_connected(*new_tip);
+		let mut notifier = ChainNotifier { header_cache: main_chain.header_cache(0..=3) };
+		let mut poller = poll::ChainPoller::new(&mut fork_chain as &mut dyn BlockSource, Network::Testnet);
+		match notifier.sync_listener(new_tip, &old_tip, &mut poller, &mut listener).await {
+			Err((e, _)) => panic!("Unexpected error: {:?}", e),
+			Ok(_) => {},
+		}
+	}
+
+	#[tokio::test]
+	async fn sync_from_longer_fork() {
+		let mut main_chain = Blockchain::default().with_height(3);
+		let mut fork_chain = main_chain.fork_at_height(1);
+		main_chain.disconnect_tip();
+
+		let new_tip = fork_chain.tip();
+		let old_tip = main_chain.tip();
+		let mut listener = MockChainListener::new()
+			.expect_block_disconnected(*old_tip)
+			.expect_block_connected(*fork_chain.at_height(2))
+			.expect_block_connected(*new_tip);
+		let mut notifier = ChainNotifier { header_cache: main_chain.header_cache(0..=2) };
+		let mut poller = poll::ChainPoller::new(&mut fork_chain as &mut dyn BlockSource, Network::Testnet);
+		match notifier.sync_listener(new_tip, &old_tip, &mut poller, &mut listener).await {
+			Err((e, _)) => panic!("Unexpected error: {:?}", e),
+			Ok(_) => {},
+		}
+	}
+
+	#[tokio::test]
+	async fn sync_from_chain_without_headers() {
+		let mut chain = Blockchain::default().with_height(3).without_headers();
+
+		let new_tip = chain.tip();
+		let old_tip = chain.at_height(1);
+		let mut listener = MockChainListener::new();
+		let mut notifier = ChainNotifier { header_cache: chain.header_cache(0..=1) };
+		let mut poller = poll::ChainPoller::new(&mut chain as &mut dyn BlockSource, Network::Testnet);
+		match notifier.sync_listener(new_tip, &old_tip, &mut poller, &mut listener).await {
+			Err((_, tip)) => assert_eq!(tip, None),
+			Ok(_) => panic!("Expected error"),
+		}
+	}
+
+	#[tokio::test]
+	async fn sync_from_chain_without_any_new_blocks() {
+		let mut chain = Blockchain::default().with_height(3).without_blocks(2..);
+
+		let new_tip = chain.tip();
+		let old_tip = chain.at_height(1);
+		let mut listener = MockChainListener::new();
+		let mut notifier = ChainNotifier { header_cache: chain.header_cache(0..=3) };
+		let mut poller = poll::ChainPoller::new(&mut chain as &mut dyn BlockSource, Network::Testnet);
+		match notifier.sync_listener(new_tip, &old_tip, &mut poller, &mut listener).await {
+			Err((_, tip)) => assert_eq!(tip, Some(old_tip)),
+			Ok(_) => panic!("Expected error"),
+		}
+	}
+
+	#[tokio::test]
+	async fn sync_from_chain_without_some_new_blocks() {
+		let mut chain = Blockchain::default().with_height(3).without_blocks(3..);
+
+		let new_tip = chain.tip();
+		let old_tip = chain.at_height(1);
+		let mut listener = MockChainListener::new()
+			.expect_block_connected(*chain.at_height(2));
+		let mut notifier = ChainNotifier { header_cache: chain.header_cache(0..=3) };
+		let mut poller = poll::ChainPoller::new(&mut chain as &mut dyn BlockSource, Network::Testnet);
+		match notifier.sync_listener(new_tip, &old_tip, &mut poller, &mut listener).await {
+			Err((_, tip)) => assert_eq!(tip, Some(chain.at_height(2))),
+			Ok(_) => panic!("Expected error"),
+		}
+	}
+}

lightning-block-sync/src/poll.rs

@@ -85,7 +85,7 @@ impl Validate for Block {
 /// A block header with validated proof of work and corresponding block hash.
 #[derive(Clone, Copy, Debug, PartialEq)]
 pub struct ValidatedBlockHeader {
-	block_hash: BlockHash,
+	pub(crate) block_hash: BlockHash,
 	inner: BlockHeaderData,
 }
 
@@ -131,7 +131,7 @@ impl ValidatedBlockHeader {
 
 /// A block with validated data against its transaction list and corresponding block hash.
 pub struct ValidatedBlock {
-	block_hash: BlockHash,
+	pub(crate) block_hash: BlockHash,
 	inner: Block,
 }