pallet-mmr: RPC API and Runtime API work with block numbers (parityte…

…ch#12345)

* pallet-mmr: RPC API works with block_numbers

* fixes

* update rpc

* fmt

* final touches in the rpc

* temporary fix

* fix

* fmt

* docs

* Update lib.rs

* use NumberFor

* validate input

* update runtime

* convert block_number to u64

* small edit

* update runtime api

* test fix

* runtime fix

* update test function

* fmt

* fix nits

* remove block_num_to_leaf_index from runtime api

* Update frame/merkle-mountain-range/src/lib.rs

Co-authored-by: Robert Hambrock <roberthambrock@gmail.com>

* fix tests

* get the code to compile after merge

* get the tests to compile

* fix in tests?

* fix test

* Update frame/merkle-mountain-range/src/tests.rs

Co-authored-by: Adrian Catangiu <adrian@parity.io>

* Update frame/merkle-mountain-range/src/lib.rs

Co-authored-by: Adrian Catangiu <adrian@parity.io>

* Update primitives/merkle-mountain-range/src/lib.rs

Co-authored-by: Adrian Catangiu <adrian@parity.io>

* fix errors & nits

* change block_num_to_leaf_index

* don't make any assumptions

* Update frame/merkle-mountain-range/src/tests.rs

Co-authored-by: Adrian Catangiu <adrian@parity.io>

* Update frame/merkle-mountain-range/src/tests.rs

Co-authored-by: Adrian Catangiu <adrian@parity.io>

* Update frame/merkle-mountain-range/src/tests.rs

Co-authored-by: Adrian Catangiu <adrian@parity.io>

* fix

* small fix

* use best_known_block_number

* best_known_block_number instead of leaves_count

* more readable?

* remove warning

* Update frame/merkle-mountain-range/src/lib.rs

Co-authored-by: Robert Hambrock <roberthambrock@gmail.com>

* simplify

* update docs

* nits

* fmt & fix

* merge fixes

* fix

* small fix

* docs & nit fixes

* Nit fixes

* remove leaf_indices_to_block_numbers()

* fmt

Co-authored-by: Robert Hambrock <roberthambrock@gmail.com>
Co-authored-by: Adrian Catangiu <adrian@parity.io>

bin/node/rpc/src/lib.rs

@@ -108,7 +108,11 @@ where
 		+ Send
 		+ 'static,
 	C::Api: substrate_frame_rpc_system::AccountNonceApi<Block, AccountId, Index>,
-	C::Api: pallet_mmr_rpc::MmrRuntimeApi<Block, <Block as sp_runtime::traits::Block>::Hash>,
+	C::Api: pallet_mmr_rpc::MmrRuntimeApi<
+		Block,
+		<Block as sp_runtime::traits::Block>::Hash,
+		BlockNumber,
+	>,
 	C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi<Block, Balance>,
 	C::Api: BabeApi<Block>,
 	C::Api: BlockBuilder<Block>,

bin/node/runtime/src/lib.rs

@@ -2016,11 +2016,15 @@ impl_runtime_apis! {
 		}
 	}
 
-	impl pallet_mmr::primitives::MmrApi<Block, mmr::Hash> for Runtime {
-		fn generate_proof(leaf_index: pallet_mmr::primitives::LeafIndex)
+	impl pallet_mmr::primitives::MmrApi<
+		Block,
+		mmr::Hash,
+		BlockNumber,
+	> for Runtime {
+		fn generate_proof(block_number: BlockNumber)
 			-> Result<(mmr::EncodableOpaqueLeaf, mmr::Proof<mmr::Hash>), mmr::Error>
 		{
-			Mmr::generate_batch_proof(vec![leaf_index]).and_then(|(leaves, proof)|
+			Mmr::generate_batch_proof(vec![block_number]).and_then(|(leaves, proof)|
 				Ok((
 					mmr::EncodableOpaqueLeaf::from_leaf(&leaves[0]),
 					mmr::BatchProof::into_single_leaf_proof(proof)?
@@ -2052,9 +2056,9 @@ impl_runtime_apis! {
 		}
 
 		fn generate_batch_proof(
-			leaf_indices: Vec<pallet_mmr::primitives::LeafIndex>,
+			block_numbers: Vec<BlockNumber>,
 		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::BatchProof<mmr::Hash>), mmr::Error> {
-			Mmr::generate_batch_proof(leaf_indices).map(|(leaves, proof)| {
+			Mmr::generate_batch_proof(block_numbers).map(|(leaves, proof)| {
 				(
 					leaves
 						.into_iter()
@@ -2066,10 +2070,10 @@ impl_runtime_apis! {
 		}
 
 		fn generate_historical_batch_proof(
-			leaf_indices: Vec<pallet_mmr::primitives::LeafIndex>,
-			leaves_count: pallet_mmr::primitives::LeafIndex,
+			block_numbers: Vec<BlockNumber>,
+			best_known_block_number: BlockNumber,
 		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::BatchProof<mmr::Hash>), mmr::Error> {
-			Mmr::generate_historical_batch_proof(leaf_indices, leaves_count).map(
+			Mmr::generate_historical_batch_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
 						leaves

client/beefy/src/lib.rs

@@ -24,7 +24,7 @@ use sc_consensus::BlockImport;
 use sc_network::ProtocolName;
 use sc_network_common::service::NetworkRequest;
 use sc_network_gossip::Network as GossipNetwork;
-use sp_api::ProvideRuntimeApi;
+use sp_api::{NumberFor, ProvideRuntimeApi};
 use sp_blockchain::HeaderBackend;
 use sp_consensus::{Error as ConsensusError, SyncOracle};
 use sp_keystore::SyncCryptoStorePtr;
@@ -200,7 +200,7 @@ where
 	C: Client<B, BE> + BlockBackend<B>,
 	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
+	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 	N: GossipNetwork<B> + NetworkRequest + SyncOracle + Send + Sync + 'static,
 {
 	let BeefyParams {

client/beefy/src/tests.rs

@@ -43,9 +43,7 @@ use beefy_primitives::{
 	KEY_TYPE as BeefyKeyType,
 };
 use sc_network::{config::RequestResponseConfig, ProtocolName};
-use sp_mmr_primitives::{
-	BatchProof, EncodableOpaqueLeaf, Error as MmrError, LeafIndex, MmrApi, Proof,
-};
+use sp_mmr_primitives::{BatchProof, EncodableOpaqueLeaf, Error as MmrError, MmrApi, Proof};
 
 use sp_api::{ApiRef, ProvideRuntimeApi};
 use sp_consensus::BlockOrigin;
@@ -247,8 +245,8 @@ macro_rules! create_test_api {
 					}
 				}
 
-				impl MmrApi<Block, MmrRootHash> for RuntimeApi {
-					fn generate_proof(_leaf_index: LeafIndex)
+				impl MmrApi<Block, MmrRootHash, NumberFor<Block>> for RuntimeApi {
+					fn generate_proof(_block_number: u64)
 						-> Result<(EncodableOpaqueLeaf, Proof<MmrRootHash>), MmrError> {
 						unimplemented!()
 					}
@@ -270,13 +268,13 @@ macro_rules! create_test_api {
 						Ok($mmr_root)
 					}
 
-					fn generate_batch_proof(_leaf_indices: Vec<LeafIndex>) -> Result<(Vec<EncodableOpaqueLeaf>, BatchProof<MmrRootHash>), MmrError> {
+					fn generate_batch_proof(_block_numbers: Vec<u64>) -> Result<(Vec<EncodableOpaqueLeaf>, BatchProof<MmrRootHash>), MmrError> {
 						unimplemented!()
 					}
 
 					fn generate_historical_batch_proof(
-						_leaf_indices: Vec<LeafIndex>,
-						_leaves_count: LeafIndex
+						_block_numbers: Vec<u64>,
+						_best_known_block_number: u64
 					) -> Result<(Vec<EncodableOpaqueLeaf>, BatchProof<MmrRootHash>), MmrError> {
 						unimplemented!()
 					}
@@ -349,7 +347,7 @@ fn initialize_beefy<API>(
 ) -> impl Future<Output = ()>
 where
 	API: ProvideRuntimeApi<Block> + Default + Sync + Send,
-	API::Api: BeefyApi<Block> + MmrApi<Block, MmrRootHash>,
+	API::Api: BeefyApi<Block> + MmrApi<Block, MmrRootHash, NumberFor<Block>>,
 {
 	let tasks = FuturesUnordered::new();
 

client/beefy/src/worker.rs

@@ -252,7 +252,7 @@ where
 	C: Client<B, BE>,
 	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
+	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 	N: NetworkEventStream + NetworkRequest + SyncOracle + Send + Sync + Clone + 'static,
 {
 	/// Return a new BEEFY worker instance.

frame/merkle-mountain-range/rpc/src/lib.rs

@@ -30,10 +30,10 @@ use jsonrpsee::{
 };
 use serde::{Deserialize, Serialize};
 
-use sp_api::ProvideRuntimeApi;
+use sp_api::{NumberFor, ProvideRuntimeApi};
 use sp_blockchain::HeaderBackend;
 use sp_core::Bytes;
-use sp_mmr_primitives::{BatchProof, Error as MmrError, LeafIndex, Proof};
+use sp_mmr_primitives::{BatchProof, Error as MmrError, Proof};
 use sp_runtime::{generic::BlockId, traits::Block as BlockT};
 
 pub use sp_mmr_primitives::MmrApi as MmrRuntimeApi;
@@ -96,61 +96,61 @@ impl<BlockHash> LeafBatchProof<BlockHash> {
 
 /// MMR RPC methods.
 #[rpc(client, server)]
-pub trait MmrApi<BlockHash> {
-	/// Generate MMR proof for given leaf index.
+pub trait MmrApi<BlockHash, BlockNumber> {
+	/// Generate MMR proof for given block number.
 	///
 	/// This method calls into a runtime with MMR pallet included and attempts to generate
-	/// MMR proof for leaf at given `leaf_index`.
+	/// MMR proof for a block with a specified `block_number`.
 	/// Optionally, a block hash at which the runtime should be queried can be specified.
 	///
 	/// Returns the (full) leaf itself and a proof for this leaf (compact encoding, i.e. hash of
 	/// the leaf). Both parameters are SCALE-encoded.
 	#[method(name = "mmr_generateProof")]
 	fn generate_proof(
 		&self,
-		leaf_index: LeafIndex,
+		block_number: BlockNumber,
 		at: Option<BlockHash>,
 	) -> RpcResult<LeafProof<BlockHash>>;
 
-	/// Generate MMR proof for the given leaf indices.
+	/// Generate MMR proof for the given block numbers.
 	///
 	/// This method calls into a runtime with MMR pallet included and attempts to generate
-	/// MMR proof for a set of leaves at the given `leaf_indices`.
+	/// MMR proof for a set of blocks with the specific `block_numbers`.
 	/// Optionally, a block hash at which the runtime should be queried can be specified.
 	///
 	/// Returns the leaves and a proof for these leaves (compact encoding, i.e. hash of
 	/// the leaves). Both parameters are SCALE-encoded.
 	/// The order of entries in the `leaves` field of the returned struct
-	/// is the same as the order of the entries in `leaf_indices` supplied
+	/// is the same as the order of the entries in `block_numbers` supplied
 	#[method(name = "mmr_generateBatchProof")]
 	fn generate_batch_proof(
 		&self,
-		leaf_indices: Vec<LeafIndex>,
+		block_numbers: Vec<BlockNumber>,
 		at: Option<BlockHash>,
 	) -> RpcResult<LeafBatchProof<BlockHash>>;
 
-	/// Generate a MMR proof for the given `leaf_indices` of the MMR that had `leaves_count` leaves.
+	/// Generate a MMR proof for the given `block_numbers` given the `best_known_block_number`.
 	///
 	/// This method calls into a runtime with MMR pallet included and attempts to generate
-	/// a MMR proof for the set of leaves at the given `leaf_indices` with MMR fixed to the state
-	/// with exactly `leaves_count` leaves. `leaves_count` must be larger than all `leaf_indices`
-	/// for the function to succeed.
+	/// a MMR proof for the set of blocks that have the given `block_numbers` with MMR given the
+	/// `best_known_block_number`. `best_known_block_number` must be larger than all the
+	/// `block_numbers` for the function to succeed.
 	///
 	/// Optionally, a block hash at which the runtime should be queried can be specified.
 	/// Note that specifying the block hash isn't super-useful here, unless you're generating
 	/// proof using non-finalized blocks where there are several competing forks. That's because
-	/// MMR state will be fixed to the state with `leaves_count`, which already points to some
-	/// historical block.
+	/// MMR state will be fixed to the state with `best_known_block_number`, which already points to
+	/// some historical block.
 	///
 	/// Returns the leaves and a proof for these leaves (compact encoding, i.e. hash of
 	/// the leaves). Both parameters are SCALE-encoded.
 	/// The order of entries in the `leaves` field of the returned struct
-	/// is the same as the order of the entries in `leaf_indices` supplied
+	/// is the same as the order of the entries in `block_numbers` supplied
 	#[method(name = "mmr_generateHistoricalBatchProof")]
 	fn generate_historical_batch_proof(
 		&self,
-		leaf_indices: Vec<LeafIndex>,
-		leaves_count: LeafIndex,
+		block_numbers: Vec<BlockNumber>,
+		best_known_block_number: BlockNumber,
 		at: Option<BlockHash>,
 	) -> RpcResult<LeafBatchProof<BlockHash>>;
 }
@@ -169,16 +169,17 @@ impl<C, B> Mmr<C, B> {
 }
 
 #[async_trait]
-impl<Client, Block, MmrHash> MmrApiServer<<Block as BlockT>::Hash> for Mmr<Client, (Block, MmrHash)>
+impl<Client, Block, MmrHash> MmrApiServer<<Block as BlockT>::Hash, NumberFor<Block>>
+	for Mmr<Client, (Block, MmrHash)>
 where
 	Block: BlockT,
 	Client: Send + Sync + 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block>,
-	Client::Api: MmrRuntimeApi<Block, MmrHash>,
+	Client::Api: MmrRuntimeApi<Block, MmrHash, NumberFor<Block>>,
 	MmrHash: Codec + Send + Sync + 'static,
 {
 	fn generate_proof(
 		&self,
-		leaf_index: LeafIndex,
+		block_number: NumberFor<Block>,
 		at: Option<<Block as BlockT>::Hash>,
 	) -> RpcResult<LeafProof<Block::Hash>> {
 		let api = self.client.runtime_api();
@@ -188,7 +189,7 @@ where
 			.generate_proof_with_context(
 				&BlockId::hash(block_hash),
 				sp_core::ExecutionContext::OffchainCall(None),
-				leaf_index,
+				block_number,
 			)
 			.map_err(runtime_error_into_rpc_error)?
 			.map_err(mmr_error_into_rpc_error)?;
@@ -198,7 +199,7 @@ where
 
 	fn generate_batch_proof(
 		&self,
-		leaf_indices: Vec<LeafIndex>,
+		block_numbers: Vec<NumberFor<Block>>,
 		at: Option<<Block as BlockT>::Hash>,
 	) -> RpcResult<LeafBatchProof<<Block as BlockT>::Hash>> {
 		let api = self.client.runtime_api();
@@ -210,7 +211,7 @@ where
 			.generate_batch_proof_with_context(
 				&BlockId::hash(block_hash),
 				sp_core::ExecutionContext::OffchainCall(None),
-				leaf_indices,
+				block_numbers,
 			)
 			.map_err(runtime_error_into_rpc_error)?
 			.map_err(mmr_error_into_rpc_error)?;
@@ -220,8 +221,8 @@ where
 
 	fn generate_historical_batch_proof(
 		&self,
-		leaf_indices: Vec<LeafIndex>,
-		leaves_count: LeafIndex,
+		block_numbers: Vec<NumberFor<Block>>,
+		best_known_block_number: NumberFor<Block>,
 		at: Option<<Block as BlockT>::Hash>,
 	) -> RpcResult<LeafBatchProof<<Block as BlockT>::Hash>> {
 		let api = self.client.runtime_api();
@@ -233,8 +234,8 @@ where
 			.generate_historical_batch_proof_with_context(
 				&BlockId::hash(block_hash),
 				sp_core::ExecutionContext::OffchainCall(None),
-				leaf_indices,
-				leaves_count,
+				block_numbers,
+				best_known_block_number,
 			)
 			.map_err(runtime_error_into_rpc_error)?
 			.map_err(mmr_error_into_rpc_error)?;

frame/merkle-mountain-range/src/lib.rs

@@ -59,7 +59,7 @@
 use codec::Encode;
 use frame_support::weights::Weight;
 use sp_runtime::{
-	traits::{self, One, Saturating},
+	traits::{self, CheckedSub, One, Saturating},
 	SaturatedConversion,
 };
 
@@ -318,37 +318,73 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 			.saturating_add(leaf_index.saturated_into())
 	}
 
-	/// Generate a MMR proof for the given `leaf_indices`.
+	/// Convert a `block_num` into a leaf index.
+	fn block_num_to_leaf_index(block_num: T::BlockNumber) -> Result<LeafIndex, primitives::Error>
+	where
+		T: frame_system::Config,
+	{
+		// leaf_idx = (leaves_count - 1) - (current_block_num - block_num);
+		let best_block_num = <frame_system::Pallet<T>>::block_number();
+		let blocks_diff = best_block_num.checked_sub(&block_num).ok_or_else(|| {
+			primitives::Error::BlockNumToLeafIndex
+				.log_debug("The provided block_number is greater than the best block number.")
+		})?;
+		let blocks_diff_as_leaf_idx = blocks_diff.try_into().map_err(|_| {
+			primitives::Error::BlockNumToLeafIndex
+				.log_debug("The `blocks_diff` couldn't be converted to `LeafIndex`.")
+		})?;
+
+		let leaf_idx = Self::mmr_leaves()
+			.checked_sub(1)
+			.and_then(|last_leaf_idx| last_leaf_idx.checked_sub(blocks_diff_as_leaf_idx))
+			.ok_or_else(|| {
+				primitives::Error::BlockNumToLeafIndex
+					.log_debug("There aren't enough leaves in the chain.")
+			})?;
+		Ok(leaf_idx)
+	}
+
+	/// Generate a MMR proof for the given `block_numbers`.
 	///
 	/// Note this method can only be used from an off-chain context
 	/// (Offchain Worker or Runtime API call), since it requires
 	/// all the leaves to be present.
 	/// It may return an error or panic if used incorrectly.
 	pub fn generate_batch_proof(
-		leaf_indices: Vec<LeafIndex>,
+		block_numbers: Vec<T::BlockNumber>,
 	) -> Result<
 		(Vec<LeafOf<T, I>>, primitives::BatchProof<<T as Config<I>>::Hash>),
 		primitives::Error,
 	> {
-		Self::generate_historical_batch_proof(leaf_indices, Self::mmr_leaves())
+		Self::generate_historical_batch_proof(
+			block_numbers,
+			<frame_system::Pallet<T>>::block_number(),
+		)
 	}
 
-	/// Generate a MMR proof for the given `leaf_indices` for the MMR of `leaves_count` size.
+	/// Generate a MMR proof for the given `block_numbers` given the `best_known_block_number`.
 	///
 	/// Note this method can only be used from an off-chain context
 	/// (Offchain Worker or Runtime API call), since it requires
 	/// all the leaves to be present.
 	/// It may return an error or panic if used incorrectly.
 	pub fn generate_historical_batch_proof(
-		leaf_indices: Vec<LeafIndex>,
-		leaves_count: LeafIndex,
+		block_numbers: Vec<T::BlockNumber>,
+		best_known_block_number: T::BlockNumber,
 	) -> Result<
 		(Vec<LeafOf<T, I>>, primitives::BatchProof<<T as Config<I>>::Hash>),
 		primitives::Error,
 	> {
-		if leaves_count > Self::mmr_leaves() {
-			return Err(Error::InvalidLeavesCount)
-		}
+		let leaves_count =
+			Self::block_num_to_leaf_index(best_known_block_number)?.saturating_add(1);
+
+		// we need to translate the block_numbers into leaf indices.
+		let leaf_indices = block_numbers
+			.iter()
+			.map(|block_num| -> Result<LeafIndex, primitives::Error> {
+				Self::block_num_to_leaf_index(*block_num)
+			})
+			.collect::<Result<Vec<LeafIndex>, _>>()?;
 
 		let mmr: ModuleMmr<mmr::storage::OffchainStorage, T, I> = mmr::Mmr::new(leaves_count);
 		mmr.generate_batch_proof(leaf_indices)