Merge branch 'develop' into feature/xyk-type-conversion

client/basic-authorship/src/basic_authorship.rs

@@ -216,27 +216,6 @@ where
             self.client
                 .new_block_at(&self.parent_id, inherent_digests, record_proof)?;
 
-        for inherent in block_builder.create_inherents(inherent_data)? {
-            match block_builder.push(inherent) {
-                Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() => {
-                    warn!("⚠️  Dropping non-mandatory inherent from overweight block.")
-                }
-                Err(ApplyExtrinsicFailed(Validity(e))) if e.was_mandatory() => {
-                    error!(
-                        "❌️ Mandatory inherent extrinsic returned error. Block cannot be produced."
-                    );
-                    Err(ApplyExtrinsicFailed(Validity(e)))?
-                }
-                Err(e) => {
-                    warn!(
-                        "❗️ Inherent extrinsic returned unexpected error: {}. Dropping.",
-                        e
-                    );
-                }
-                Ok(_) => {}
-            }
-        }
-
         // proceed with transactions
         let block_timer = time::Instant::now();
         let mut skipped = 0;
@@ -302,7 +281,29 @@ where
 
         self.transaction_pool.remove_invalid(&unqueue_invalid);
 
-        let (block, storage_changes, proof) = block_builder.build()?.into_inner();
+        let (seed, inherents) = block_builder.create_inherents(inherent_data)?;
+        for inherent in inherents {
+            match block_builder.push(inherent) {
+                Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() => {
+                    warn!("⚠️  Dropping non-mandatory inherent from overweight block.")
+                }
+                Err(ApplyExtrinsicFailed(Validity(e))) if e.was_mandatory() => {
+                    error!(
+                        "❌️ Mandatory inherent extrinsic returned error. Block cannot be produced."
+                    );
+                    Err(ApplyExtrinsicFailed(Validity(e)))?
+                }
+                Err(e) => {
+                    warn!(
+                        "❗️ Inherent extrinsic returned unexpected error: {}. Dropping.",
+                        e
+                    );
+                }
+                Ok(_) => {}
+            }
+        }
+
+        let (block, storage_changes, proof) = block_builder.build(seed)?.into_inner();
 
         self.metrics.report(|metrics| {
             metrics

client/block-builder/Cargo.toml

@@ -28,6 +28,7 @@ sc-client-api = "2.0.0"
 codec = { package = "parity-scale-codec", version = "1.3.4", features = ["derive"] }
 extrinsic-info-runtime-api = { path='../../primitives/extrinsic-info-runtime-api', version='2.0.0' }
 extrinsic-shuffler = { path='../shuffler', version='0.8.0' }
+pallet-random-seed = { path='../../pallets/random-seed', version='2.0.0' }
 rand = "0.7.3"
 
 [dev-dependencies]

client/block-builder/src/lib.rs

@@ -45,6 +45,10 @@ use sp_runtime::{
     },
 };
 
+use pallet_random_seed::RandomSeedInherentDataProvider;
+use sp_core::H256;
+use sp_inherents::ProvideInherentData;
+
 pub use sp_block_builder::BlockBuilder as BlockBuilderApi;
 
 use sc_client_api::backend;
@@ -169,20 +173,21 @@ where
     ///
     /// This will ensure the extrinsic can be validly executed (by executing it).
     pub fn push(&mut self, xt: <Block as BlockT>::Extrinsic) -> Result<(), ApiErrorFor<A, Block>> {
-        info!("Pushing transactions without execution");
         self.extrinsics.push(xt);
         Ok(())
+        // TODO: check if its possible to verify transaction by first
+        // applying all the transactions from current block and then applying
+        // particular one from passed to BlockBuilder::push as in origin implementation
 
-        // info!("Going to call api tx execution");
         // self.api.execute_in_transaction(|api| {
         // 	match api.apply_extrinsic_with_context(
-        // 		block_id,
+        // 		&block_id,
         // 		ExecutionContext::BlockConstruction,
         // 		xt.clone(),
         // 	) {
         // 		Ok(Ok(_)) => {
-        // 			extrinsics.push(xt);
-        // 			TransactionOutcome::Commit(Ok(()))
+        // 		exts.push(xt.clone());
+        // 			TransactionOutcome::Rollback(Ok(()))
         // 		}
         // 		Ok(Err(tx_validity)) => {
         // 			TransactionOutcome::Rollback(
@@ -201,12 +206,12 @@ where
     /// The storage proof will be `Some(_)` when proof recording was enabled.
     pub fn build(
         mut self,
+        seed: H256,
     ) -> Result<BuiltBlock<Block, backend::StateBackendFor<B, Block>>, ApiErrorFor<A, Block>> {
-        let block_id = &self.block_id;
-
         let extrinsics = self.extrinsics.clone();
         let parent_hash = self.parent_hash;
-        let extrinsics_hash = BlakeTwo256::hash(&extrinsics.encode());
+
+        let block_id = &self.block_id;
 
         match self
             .backend
@@ -237,7 +242,7 @@ where
                         &self.api,
                         &self.block_id,
                         previous_block_extrinsics,
-                        extrinsics_hash,
+                        seed,
                     );
 
                     for xt in shuffled_extrinsics.iter() {
@@ -296,18 +301,26 @@ where
     /// Returns the inherents created by the runtime or an error if something failed.
     pub fn create_inherents(
         &mut self,
-        inherent_data: sp_inherents::InherentData,
-    ) -> Result<Vec<Block::Extrinsic>, ApiErrorFor<A, Block>> {
-        let block_id = self.block_id;
-        self.api.execute_in_transaction(move |api| {
-            // `create_inherents` should not change any state, to ensure this we always rollback
-            // the transaction.
-            TransactionOutcome::Rollback(api.inherent_extrinsics_with_context(
-                &block_id,
-                ExecutionContext::BlockConstruction,
-                inherent_data,
-            ))
-        })
+        mut inherent_data: sp_inherents::InherentData,
+    ) -> Result<(H256, Vec<Block::Extrinsic>), ApiErrorFor<A, Block>> {
+        let block_id = self.block_id.clone();
+        // Result<(H256,Vec<Block::Extrinsic>), ApiErrorFor<A, Block>> {
+        let seed = BlakeTwo256::hash(&self.extrinsics.encode());
+        RandomSeedInherentDataProvider(seed)
+            .provide_inherent_data(&mut inherent_data)
+            .unwrap();
+
+        self.api
+            .execute_in_transaction(move |api| {
+                // `create_inherents` should not change any state, to ensure this we always rollback
+                // the transaction.
+                TransactionOutcome::Rollback(api.inherent_extrinsics_with_context(
+                    &block_id,
+                    ExecutionContext::BlockConstruction,
+                    inherent_data,
+                ))
+            })
+            .map(|inherents| (seed, inherents))
     }
 }
 

client/service/src/client/client.rs

@@ -930,7 +930,14 @@ where
 						} else {
 
 							info!("previous block has extrinsics");
-							let extrinsics_hash = BlakeTwo256::hash(&body.clone().encode());
+                            let mut this_block_extrinsics = body.clone();
+                            // TODO that step will be obsolete when better randomness source will
+                            // be used
+                            this_block_extrinsics.pop().unwrap(); // remove inherent
+                            this_block_extrinsics.pop().unwrap(); // remove inherent
+                            // TODO some extra check that compares calculated seed with one from
+                            // stroed in inherent would be nice idea
+							let extrinsics_hash = BlakeTwo256::hash(&this_block_extrinsics.encode());
                             let shuffled_extrinsics = extrinsic_shuffler::shuffle::<Block, Self>(&runtime_api, &at,previous_block_extrinsics, extrinsics_hash);
 
 							runtime_api.execute_block_with_context(

client/shuffler/Cargo.toml

@@ -21,7 +21,6 @@ sp-core = "2.0.0"
 sp-std = { default-features = false, version = '2.0.0' }
 codec = { package = "parity-scale-codec", version = "1.3.4", features = ["derive"] }
 extrinsic-info-runtime-api = { path='../../primitives/extrinsic-info-runtime-api', version='2.0.0' }
-rand = "0.7.3"
 
 [dev-dependencies]
 

client/shuffler/src/lib.rs

@@ -1,8 +1,5 @@
 #![cfg_attr(not(feature = "std"), no_std)]
 use extrinsic_info_runtime_api::runtime_api::ExtrinsicInfoRuntimeApi;
-use rand::prelude::SliceRandom;
-use rand::rngs::StdRng;
-use rand::SeedableRng;
 use sp_api::{ApiExt, ApiRef, Encode, HashT, ProvideRuntimeApi, TransactionOutcome};
 use sp_core::crypto::Ss58Codec;
 use sp_core::H256;
@@ -11,26 +8,85 @@ use sp_runtime::traits::{BlakeTwo256, Block as BlockT};
 use sp_runtime::AccountId32;
 use sp_std::collections::btree_map::BTreeMap;
 use sp_std::collections::vec_deque::VecDeque;
+use sp_std::convert::TryInto;
 use sp_std::vec::Vec;
 
+pub struct Xoshiro256PlusPlus {
+    s: [u64; 4],
+}
+
+fn rotl(x: u64, k: u64) -> u64 {
+    ((x) << (k)) | ((x) >> (64 - (k)))
+}
+
+impl Xoshiro256PlusPlus {
+    #[inline]
+    fn from_seed(seed: [u8; 32]) -> Xoshiro256PlusPlus {
+        Xoshiro256PlusPlus {
+            s: [
+                u64::from_le_bytes(seed[0..8].try_into().unwrap()),
+                u64::from_le_bytes(seed[8..16].try_into().unwrap()),
+                u64::from_le_bytes(seed[16..24].try_into().unwrap()),
+                u64::from_le_bytes(seed[24..32].try_into().unwrap()),
+            ],
+        }
+    }
+
+    fn next_u32(&mut self) -> u32 {
+        let t: u64 = self.s[1] << 17;
+
+        self.s[2] ^= self.s[0];
+        self.s[3] ^= self.s[1];
+        self.s[1] ^= self.s[2];
+        self.s[0] ^= self.s[3];
+
+        self.s[2] ^= t;
+
+        self.s[3] = rotl(self.s[3], 45);
+
+        return (self.s[0].wrapping_add(self.s[3])) as u32;
+    }
+}
+
+/// In order to be able to recreate shuffling order anywere lets use
+/// explicit algorithms
+/// - Xoshiro256StarStar as random number generator
+/// - Fisher-Yates variation as shuffling algorithm
+///
+/// ref https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle
+///
+/// To shuffle an array a of n elements (indices 0..n-1):
+///
+/// for i from n−1 downto 1 do
+///     j ← random integer such that 0 ≤ j ≤ i
+///     exchange a[j] and a[i]
+///
+fn fisher_yates<T>(data: &mut Vec<T>, seed: H256) {
+    let mut s = Xoshiro256PlusPlus::from_seed(seed.into());
+    for i in (1..(data.len())).rev() {
+        let j = s.next_u32() % (i as u32);
+        data.swap(i, j as usize);
+    }
+}
+
 /// shuffles extrinsics assuring that extrinsics signed by single account will be still evaluated
 /// in proper order
 pub fn shuffle<'a, Block, Api>(
     api: &ApiRef<'a, Api::Api>,
     block_id: &BlockId<Block>,
     extrinsics: Vec<Block::Extrinsic>,
-    hash: H256,
+    seed: H256,
 ) -> Vec<Block::Extrinsic>
 where
     Block: BlockT,
     Api: ProvideRuntimeApi<Block> + 'a,
     Api::Api: ExtrinsicInfoRuntimeApi<Block>,
 {
-    log::debug!(target: "block_shuffler", "shuffling extrinsics with seed: {:#X}", hash);
+    log::debug!(target: "block_shuffler", "shuffling extrinsics with seed: {:#X} => {}", seed, Xoshiro256PlusPlus::from_seed(seed.into()).next_u32() );
 
-    let mut grouped_extrinsics: BTreeMap<Option<AccountId32>, VecDeque<_>> = extrinsics
+    let extrinsics: Vec<(Option<AccountId32>, Block::Extrinsic)> = extrinsics
         .into_iter()
-        .fold(BTreeMap::new(), |mut groups, tx| {
+        .map(|tx| {
             let tx_hash = BlakeTwo256::hash(&tx.encode());
             let who = api.execute_in_transaction(|api| {
                 // store deserialized data and revert state modification caused by 'get_info' call
@@ -41,25 +97,24 @@ where
             })
             .expect("extrinsic deserialization should not fail!")
             .map(|info| Some(info.who)).unwrap_or(None);
-
             log::debug!(target: "block_shuffler", "who:{:48}  extrinsic:{:?}",who.clone().map(|x| x.to_ss58check()).unwrap_or_else(|| String::from("None")), tx_hash);
+            (who, tx)
+        }).collect();
+
+    // generate exact number of slots for each account
+    // [ Alice, Alice, Alice, ... , Bob, Bob, Bob, ... ]
+    let mut slots: Vec<Option<AccountId32>> =
+        extrinsics.iter().map(|(who, _)| who).cloned().collect();
 
+    let mut grouped_extrinsics: BTreeMap<Option<AccountId32>, VecDeque<_>> = extrinsics
+        .into_iter()
+        .fold(BTreeMap::new(), |mut groups, (who, tx)| {
             groups.entry(who).or_insert(VecDeque::new()).push_back(tx);
             groups
         });
 
-    // generate exact number of slots for each account
-    // [ Alice, Alice, Alice, ... , Bob, Bob, Bob, ... ]
-    let mut slots: Vec<_> = grouped_extrinsics
-        .iter()
-        .map(|(who, txs)| vec![who; txs.len()])
-        .flatten()
-        .map(|elem| elem.to_owned())
-        .collect();
-
     // shuffle slots
-    let mut seed: StdRng = SeedableRng::from_seed(hash.to_fixed_bytes());
-    slots.shuffle(&mut seed);
+    fisher_yates(&mut slots, seed);
 
     // fill slots using extrinsics in order
     // [ Alice, Bob, ... , Alice, Bob ]