Add pluggable BEEFY payload constructors (paritytech#12428)

* primitives/beefy: move Payload to its own file

* primitives/beefy: add Payload tests

* primitives/beefy: add MmrRootProvider as custom BEEFY payload provider

* client/beefy: use generic BEEFY 'PayloadProvider'

* primitives/beefy: rename Payload::new to Payload::from_single_entry for clarity

* fix visibility

* fix cargo doc

client/beefy/rpc/src/lib.rs

@@ -170,7 +170,7 @@ mod tests {
  communication::notification::BeefyVersionedFinalityProofSender,
  justification::BeefyVersionedFinalityProof,
  };
- use beefy_primitives::{known_payload_ids, Payload, SignedCommitment};
+ use beefy_primitives::{known_payloads, Payload, SignedCommitment};
  use codec::{Decode, Encode};
  use jsonrpsee::{types::EmptyParams, RpcModule};
  use sp_runtime::traits::{BlakeTwo256, Hash};
@@ -266,7 +266,8 @@ mod tests {
  }
 
  fn create_finality_proof() -> BeefyVersionedFinalityProof<Block> {
- let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+ let payload =
+ Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
  BeefyVersionedFinalityProof::<Block>::V1(SignedCommitment {
  commitment: beefy_primitives::Commitment {
  payload,

client/beefy/src/communication/gossip.rs

@@ -237,8 +237,7 @@ mod tests {
 
  use crate::keystore::{tests::Keyring, BeefyKeystore};
  use beefy_primitives::{
- crypto::Signature, known_payload_ids, Commitment, MmrRootHash, Payload, VoteMessage,
- KEY_TYPE,
+ crypto::Signature, known_payloads, Commitment, MmrRootHash, Payload, VoteMessage, KEY_TYPE,
  };
 
  use super::*;
@@ -348,7 +347,10 @@ mod tests {
  }
 
  fn dummy_vote(block_number: u64) -> VoteMessage<u64, Public, Signature> {
- let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, MmrRootHash::default().encode());
+ let payload = Payload::from_single_entry(
+ known_payloads::MMR_ROOT_ID,
+ MmrRootHash::default().encode(),
+ );
  let commitment = Commitment { payload, block_number, validator_set_id: 0 };
  let signature = sign_commitment(&Keyring::Alice, &commitment);
 

client/beefy/src/justification.rs

@@ -81,7 +81,7 @@ fn verify_with_validator_set<Block: BlockT>(
 #[cfg(test)]
 pub(crate) mod tests {
  use beefy_primitives::{
- known_payload_ids, Commitment, Payload, SignedCommitment, VersionedFinalityProof,
+ known_payloads, Commitment, Payload, SignedCommitment, VersionedFinalityProof,
  };
  use substrate_test_runtime_client::runtime::Block;
 
@@ -94,7 +94,7 @@ pub(crate) mod tests {
  keys: &[Keyring],
  ) -> BeefyVersionedFinalityProof<Block> {
  let commitment = Commitment {
- payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+ payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
  block_number: block_num,
  validator_set_id: validator_set.id(),
  };

client/beefy/src/lib.rs

@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use beefy_primitives::{BeefyApi, MmrRootHash};
+use beefy_primitives::{BeefyApi, MmrRootHash, PayloadProvider};
 use parking_lot::Mutex;
 use prometheus::Registry;
 use sc_client_api::{Backend, BlockBackend, BlockchainEvents, Finalizer};
@@ -167,11 +167,13 @@ pub struct BeefyNetworkParams<B: Block, N> {
 }
 
 /// BEEFY gadget initialization parameters.
-pub struct BeefyParams<B: Block, BE, C, N, R> {
+pub struct BeefyParams<B: Block, BE, C, N, P, R> {
  /// BEEFY client
  pub client: Arc<C>,
  /// Client Backend
  pub backend: Arc<BE>,
+ /// BEEFY Payload provider
+ pub payload_provider: P,
  /// Runtime Api Provider
  pub runtime: Arc<R>,
  /// Local key store
@@ -191,18 +193,20 @@ pub struct BeefyParams<B: Block, BE, C, N, R> {
 /// Start the BEEFY gadget.
 ///
 /// This is a thin shim around running and awaiting a BEEFY worker.
-pub async fn start_beefy_gadget<B, BE, C, N, R>(beefy_params: BeefyParams<B, BE, C, N, R>)
+pub async fn start_beefy_gadget<B, BE, C, N, P, R>(beefy_params: BeefyParams<B, BE, C, N, P, R>)
 where
  B: Block,
  BE: Backend<B>,
  C: Client<B, BE> + BlockBackend<B>,
+ P: PayloadProvider<B>,
  R: ProvideRuntimeApi<B>,
  R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
  N: GossipNetwork<B> + NetworkRequest + SyncOracle + Send + Sync + 'static,
 {
  let BeefyParams {
  client,
  backend,
+ payload_provider,
  runtime,
  key_store,
  network_params,
@@ -249,6 +253,7 @@ where
  let worker_params = worker::WorkerParams {
  client,
  backend,
+ payload_provider,
  runtime,
  network,
  key_store: key_store.into(),
@@ -261,7 +266,7 @@ where
  min_block_delta,
  };
 
- let worker = worker::BeefyWorker::<_, _, _, _, _>::new(worker_params);
+ let worker = worker::BeefyWorker::<_, _, _, _, _, _>::new(worker_params);
 
  futures::future::join(worker.run(), on_demand_justifications_handler.run()).await;
 }

client/beefy/src/tests.rs

@@ -38,6 +38,7 @@ use sc_utils::notification::NotificationReceiver;
 
 use beefy_primitives::{
  crypto::{AuthorityId, Signature},
+ mmr::MmrRootProvider,
  BeefyApi, ConsensusLog, MmrRootHash, ValidatorSet, VersionedFinalityProof, BEEFY_ENGINE_ID,
  KEY_TYPE as BeefyKeyType,
 };
@@ -372,10 +373,12 @@ where
  justifications_protocol_name: on_demand_justif_handler.protocol_name(),
  _phantom: PhantomData,
  };
+ let payload_provider = MmrRootProvider::new(api.clone());
 
  let beefy_params = crate::BeefyParams {
  client: peer.client().as_client(),
  backend: peer.client().as_backend(),
+ payload_provider,
  runtime: api.clone(),
  key_store: Some(keystore),
  network_params,
@@ -384,7 +387,7 @@ where
  prometheus_registry: None,
  on_demand_justifications_handler: on_demand_justif_handler,
  };
- let task = crate::start_beefy_gadget::<_, _, _, _, _>(beefy_params);
+ let task = crate::start_beefy_gadget::<_, _, _, _, _, _>(beefy_params);
 
  fn assert_send<T: Send>(_: &T) {}
  assert_send(&task);

client/beefy/src/worker.rs

@@ -48,7 +48,7 @@ use sp_runtime::{
 
 use beefy_primitives::{
  crypto::{AuthorityId, Signature},
- known_payload_ids, BeefyApi, Commitment, ConsensusLog, MmrRootHash, Payload, SignedCommitment,
+ BeefyApi, Commitment, ConsensusLog, MmrRootHash, Payload, PayloadProvider, SignedCommitment,
  ValidatorSet, VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID, GENESIS_AUTHORITY_SET_ID,
 };
 
@@ -194,9 +194,10 @@ impl<B: Block> VoterOracle<B> {
  }
 }
 
-pub(crate) struct WorkerParams<B: Block, BE, C, R, N> {
+pub(crate) struct WorkerParams<B: Block, BE, C, P, R, N> {
  pub client: Arc<C>,
  pub backend: Arc<BE>,
+ pub payload_provider: P,
  pub runtime: Arc<R>,
  pub network: N,
  pub key_store: BeefyKeystore,
@@ -210,10 +211,11 @@ pub(crate) struct WorkerParams<B: Block, BE, C, R, N> {
 }
 
 /// A BEEFY worker plays the BEEFY protocol
-pub(crate) struct BeefyWorker<B: Block, BE, C, R, N> {
+pub(crate) struct BeefyWorker<B: Block, BE, C, P, R, N> {
  // utilities
  client: Arc<C>,
  backend: Arc<BE>,
+ payload_provider: P,
  runtime: Arc<R>,
  network: N,
  key_store: BeefyKeystore,
@@ -243,11 +245,12 @@ pub(crate) struct BeefyWorker<B: Block, BE, C, R, N> {
  voting_oracle: VoterOracle<B>,
 }
 
-impl<B, BE, C, R, N> BeefyWorker<B, BE, C, R, N>
+impl<B, BE, C, P, R, N> BeefyWorker<B, BE, C, P, R, N>
 where
  B: Block + Codec,
  BE: Backend<B>,
  C: Client<B, BE>,
+ P: PayloadProvider<B>,
  R: ProvideRuntimeApi<B>,
  R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
  N: NetworkEventStream + NetworkRequest + SyncOracle + Send + Sync + Clone + 'static,
@@ -258,10 +261,11 @@ where
  /// BEEFY pallet has been deployed on-chain.
  ///
  /// The BEEFY pallet is needed in order to keep track of the BEEFY authority set.
- pub(crate) fn new(worker_params: WorkerParams<B, BE, C, R, N>) -> Self {
+ pub(crate) fn new(worker_params: WorkerParams<B, BE, C, P, R, N>) -> Self {
  let WorkerParams {
  client,
  backend,
+ payload_provider,
  runtime,
  key_store,
  network,
@@ -282,6 +286,7 @@ where
  BeefyWorker {
  client: client.clone(),
  backend,
+ payload_provider,
  runtime,
  network,
  known_peers,
@@ -299,17 +304,6 @@ where
  }
  }
 
- /// Simple wrapper that gets MMR root from header digests or from client state.
- fn get_mmr_root_digest(&self, header: &B::Header) -> Option<MmrRootHash> {
- find_mmr_root_digest::<B>(header).or_else(|| {
- self.runtime
- .runtime_api()
- .mmr_root(&BlockId::hash(header.hash()))
- .ok()
- .and_then(|r| r.ok())
- })
- }
-
  /// Verify `active` validator set for `block` against the key store
  ///
  /// We want to make sure that we have _at least one_ key in our keystore that
@@ -621,13 +615,12 @@ where
  };
  let target_hash = target_header.hash();
 
- let mmr_root = if let Some(hash) = self.get_mmr_root_digest(&target_header) {
+ let payload = if let Some(hash) = self.payload_provider.payload(&target_header) {
  hash
  } else {
  warn!(target: "beefy", "🥩 No MMR root digest found for: {:?}", target_hash);
  return Ok(())
  };
- let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, mmr_root.encode());
 
  let rounds = self.voting_oracle.rounds_mut().ok_or(Error::UninitSession)?;
  if !rounds.should_self_vote(&(payload.clone(), target_number)) {
@@ -917,20 +910,6 @@ where
  }
 }
 
-/// Extract the MMR root hash from a digest in the given header, if it exists.
-fn find_mmr_root_digest<B>(header: &B::Header) -> Option<MmrRootHash>
-where
- B: Block,
-{
- let id = OpaqueDigestItemId::Consensus(&BEEFY_ENGINE_ID);
-
- let filter = |log: ConsensusLog<AuthorityId>| match log {
- ConsensusLog::MmrRoot(root) => Some(root),
- _ => None,
- };
- header.digest().convert_first(|l| l.try_to(id).and_then(filter))
-}
-
 /// Scan the `header` digest log for a BEEFY validator set change. Return either the new
 /// validator set or `None` in case no validator set change has been signaled.
 fn find_authorities_change<B>(header: &B::Header) -> Option<ValidatorSet<AuthorityId>>
@@ -1016,8 +995,8 @@ pub(crate) mod tests {
  BeefyRPCLinks,
  };
 
+ use beefy_primitives::{known_payloads, mmr::MmrRootProvider};
  use futures::{executor::block_on, future::poll_fn, task::Poll};
-
  use sc_client_api::{Backend as BackendT, HeaderBackend};
  use sc_network::NetworkService;
  use sc_network_test::{PeersFullClient, TestNetFactory};
@@ -1032,7 +1011,14 @@ pub(crate) mod tests {
  peer: &BeefyPeer,
  key: &Keyring,
  min_block_delta: u32,
- ) -> BeefyWorker<Block, Backend, PeersFullClient, TestApi, Arc<NetworkService<Block, H256>>> {
+ ) -> BeefyWorker<
+ Block,
+ Backend,
+ PeersFullClient,
+ MmrRootProvider<Block, TestApi>,
+ TestApi,
+ Arc<NetworkService<Block, H256>>,
+ > {
  let keystore = create_beefy_keystore(*key);
 
  let (to_rpc_justif_sender, from_voter_justif_stream) =
@@ -1064,9 +1050,11 @@ pub(crate) mod tests {
  "/beefy/justifs/1".into(),
  known_peers.clone(),
  );
+ let payload_provider = MmrRootProvider::new(api.clone());
  let worker_params = crate::worker::WorkerParams {
  client: peer.client().as_client(),
  backend: peer.client().as_backend(),
+ payload_provider,
  runtime: api,
  key_store: Some(keystore).into(),
  known_peers,
@@ -1078,7 +1066,7 @@ pub(crate) mod tests {
  network,
  on_demand_justifications,
  };
- BeefyWorker::<_, _, _, _, _>::new(worker_params)
+ BeefyWorker::<_, _, _, _, _, _>::new(worker_params)
  }
 
  #[test]
@@ -1300,30 +1288,6 @@ pub(crate) mod tests {
  assert_eq!(extracted, Some(validator_set));
  }
 
- #[test]
- fn extract_mmr_root_digest() {
- let mut header = Header::new(
- 1u32.into(),
- Default::default(),
- Default::default(),
- Default::default(),
- Digest::default(),
- );
-
- // verify empty digest shows nothing
- assert!(find_mmr_root_digest::<Block>(&header).is_none());
-
- let mmr_root_hash = H256::random();
- header.digest_mut().push(DigestItem::Consensus(
- BEEFY_ENGINE_ID,
- ConsensusLog::<AuthorityId>::MmrRoot(mmr_root_hash).encode(),
- ));
-
- // verify validator set is correctly extracted from digest
- let extracted = find_mmr_root_digest::<Block>(&header);
- assert_eq!(extracted, Some(mmr_root_hash));
- }
-
  #[test]
  fn keystore_vs_validator_set() {
  let keys = &[Keyring::Alice];
@@ -1363,7 +1327,7 @@ pub(crate) mod tests {
 
  let create_finality_proof = |block_num: NumberFor<Block>| {
  let commitment = Commitment {
- payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+ payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
  block_number: block_num,
  validator_set_id: validator_set.id(),
  };
@@ -1482,7 +1446,7 @@ pub(crate) mod tests {
  block_number: NumberFor<Block>,
  ) -> VoteMessage<NumberFor<Block>, AuthorityId, Signature> {
  let commitment = Commitment {
- payload: Payload::new(*b"BF", vec![]),
+ payload: Payload::from_single_entry(*b"BF", vec![]),
  block_number,
  validator_set_id: 0,
  };
@@ -1574,7 +1538,7 @@ pub(crate) mod tests {
 
  // import/append BEEFY justification for session boundary block 10
  let commitment = Commitment {
- payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+ payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
  block_number: 10,
  validator_set_id: validator_set.id(),
  };
@@ -1608,7 +1572,7 @@ pub(crate) mod tests {
 
  // import/append BEEFY justification for block 12
  let commitment = Commitment {
- payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+ payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
  block_number: 12,
  validator_set_id: validator_set.id(),
  };