Merge branch 'master' of github.com:paritytech/substrate into a-tx-st…

…orage2

.maintain/monitoring/alerting-rules/alerting-rule-tests.yaml

@@ -175,7 +175,7 @@ tests:
                       polkadot-abcdef01234-abcdef has been monotonically
                       increasing for more than 10 minutes."
                 - exp_labels:
-                      severity: critical
+                      severity: warning
                       pod: polkadot-abcdef01234-abcdef
                       instance: polkadot-abcdef01234-abcdef
                       job: polkadot
@@ -190,7 +190,7 @@ tests:
             # same. Thus expect an alert.
             exp_alerts:
                 - exp_labels:
-                      severity: critical
+                      severity: warning
                       pod: polkadot-abcdef01234-abcdef
                       instance: polkadot-abcdef01234-abcdef
                       job: polkadot

.maintain/monitoring/alerting-rules/alerting-rules.yaml

@@ -74,7 +74,7 @@ groups:
     increase(polkadot_sub_txpool_validations_finished[5m]) > 0'
     for: 30m
     labels:
-      severity: critical
+      severity: warning
     annotations:
       message: 'The transaction pool size on node {{ $labels.instance }} has
       been monotonically increasing for more than 30 minutes.'
@@ -83,7 +83,7 @@ groups:
     polkadot_sub_txpool_validations_finished > 10000'
     for: 5m
     labels:
-      severity: critical
+      severity: warning
     annotations:
       message: 'The transaction pool size on node {{ $labels.instance }} has
       been above 10_000 for more than 5 minutes.'

client/api/src/backend.rs

@@ -475,6 +475,12 @@ pub trait Backend<Block: BlockT>: AuxStore + Send + Sync {
 		revert_finalized: bool,
 	) -> sp_blockchain::Result<(NumberFor<Block>, HashSet<Block::Hash>)>;
 
+	/// Discard non-best, unfinalized leaf block.
+	fn remove_leaf_block(
+		&self,
+		hash: &Block::Hash,
+	) -> sp_blockchain::Result<()>;
+
 	/// Insert auxiliary data into key-value store.
 	fn insert_aux<
 		'a,

client/api/src/in_mem.rs

@@ -777,6 +777,13 @@ impl<Block: BlockT> backend::Backend<Block> for Backend<Block> where Block::Hash
 		Ok((Zero::zero(), HashSet::new()))
 	}
 
+	fn remove_leaf_block(
+		&self,
+		_hash: &Block::Hash,
+	) -> sp_blockchain::Result<()> {
+		Ok(())
+	}
+
 	fn get_import_lock(&self) -> &RwLock<()> {
 		&self.import_lock
 	}

client/api/src/leaves.rs

@@ -216,8 +216,8 @@ impl<H, N> LeafSet<H, N> where
 		self.pending_removed.clear();
 	}
 
-	#[cfg(test)]
-	fn contains(&self, number: N, hash: H) -> bool {
+	/// Check if given block is a leaf.
+	pub fn contains(&self, number: N, hash: H) -> bool {
 		self.storage.get(&Reverse(number)).map_or(false, |hashes| hashes.contains(&hash))
 	}
 

client/consensus/slots/src/slots.rs

@@ -40,11 +40,12 @@ pub fn duration_now() -> Duration {
 }
 
 /// Returns the duration until the next slot from now.
-pub fn time_until_next(slot_duration: Duration) -> Duration {
-	let remaining_full_millis = slot_duration.as_millis()
-		- (duration_now().as_millis() % slot_duration.as_millis())
-		- 1;
-	Duration::from_millis(remaining_full_millis as u64)
+pub fn time_until_next_slot(slot_duration: Duration) -> Duration {
+	let now = duration_now().as_millis();
+
+	let next_slot = (now + slot_duration.as_millis()) / slot_duration.as_millis();
+	let remaining_millis = next_slot * slot_duration.as_millis() - now;
+	Duration::from_millis(remaining_millis as u64)
 }
 
 /// Information about a slot.
@@ -86,7 +87,7 @@ impl<B: BlockT> SlotInfo<B> {
 			duration,
 			chain_head,
 			block_size_limit,
-			ends_at: Instant::now() + time_until_next(duration),
+			ends_at: Instant::now() + time_until_next_slot(duration),
 		}
 	}
 }
@@ -132,7 +133,7 @@ where
 			self.inner_delay = match self.inner_delay.take() {
 				None => {
 					// schedule wait.
-					let wait_dur = time_until_next(self.slot_duration);
+					let wait_dur = time_until_next_slot(self.slot_duration);
 					Some(Delay::new(wait_dur))
 				}
 				Some(d) => Some(d),
@@ -143,7 +144,12 @@ where
 			}
 			// timeout has fired.
 
-			let ends_at = Instant::now() + time_until_next(self.slot_duration);
+			let ends_in = time_until_next_slot(self.slot_duration);
+
+			// reschedule delay for next slot.
+			self.inner_delay = Some(Delay::new(ends_in));
+
+			let ends_at = Instant::now() + ends_in;
 
 			let chain_head = match self.client.best_chain() {
 				Ok(x) => x,
@@ -174,10 +180,6 @@ where
 			let slot = inherent_data_providers.slot();
 			let inherent_data = inherent_data_providers.create_inherent_data()?;
 
-			// reschedule delay for next slot.
-			let ends_in = time_until_next(self.slot_duration);
-			self.inner_delay = Some(Delay::new(ends_in));
-
 			// never yield the same slot twice.
 			if slot > self.last_slot {
 				self.last_slot = slot;

client/db/src/lib.rs

@@ -654,6 +654,7 @@ impl<Block: BlockT> HeaderMetadata<Block> for BlockchainDb<Block> {
 	}
 
 	fn remove_header_metadata(&self, hash: Block::Hash) {
+		self.header_cache.lock().remove(&hash);
 		self.header_metadata_cache.remove_header_metadata(hash);
 	}
 }
@@ -2002,6 +2003,59 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
 		Ok((reverted, reverted_finalized))
 	}
 
+	fn remove_leaf_block(
+		&self,
+		hash: &Block::Hash,
+	) -> ClientResult<()> {
+		let best_hash = self.blockchain.info().best_hash;
+
+		if best_hash == *hash {
+			return Err(
+				sp_blockchain::Error::Backend(
+					format!("Can't remove best block {:?}", hash)
+				)
+			)
+		}
+
+		let hdr = self.blockchain.header_metadata(hash.clone())?;
+		if !self.have_state_at(&hash, hdr.number) {
+			return Err(
+				sp_blockchain::Error::UnknownBlock(
+					format!("State already discarded for {:?}", hash)
+				)
+			)
+		}
+
+		let mut leaves = self.blockchain.leaves.write();
+		if !leaves.contains(hdr.number, *hash) {
+			return Err(
+				sp_blockchain::Error::Backend(
+					format!("Can't remove non-leaf block {:?}", hash)
+				)
+			)
+		}
+
+		let mut transaction = Transaction::new();
+		if let Some(commit) = self.storage.state_db.remove(hash) {
+			apply_state_commit(&mut transaction, commit);
+		}
+		transaction.remove(columns::KEY_LOOKUP, hash.as_ref());
+		let changes_trie_cache_ops = self.changes_tries_storage.revert(
+			&mut transaction,
+			&cache::ComplexBlockId::new(
+				*hash,
+				hdr.number,
+			),
+		)?;
+
+		self.changes_tries_storage.post_commit(Some(changes_trie_cache_ops));
+		leaves.revert(hash.clone(), hdr.number);
+		leaves.prepare_transaction(&mut transaction, columns::META, meta_keys::LEAF_PREFIX);
+		self.storage.db.commit(transaction)?;
+		self.blockchain().remove_header_metadata(*hash);
+		Ok(())
+	}
+
 	fn blockchain(&self) -> &BlockchainDb<Block> {
 		&self.blockchain
 	}
@@ -3041,4 +3095,36 @@ pub(crate) mod tests {
 			}
 		}
 	}
+
+	#[test]
+	fn remove_leaf_block_works() {
+		let backend = Backend::<Block>::new_test_with_tx_storage(
+			2,
+			10,
+			TransactionStorageMode::StorageChain
+		);
+		let mut blocks = Vec::new();
+		let mut prev_hash = Default::default();
+		for i in 0 .. 2 {
+			let hash = insert_block(&backend, i, prev_hash, None, Default::default(), vec![i.into()], None);
+			blocks.push(hash);
+			prev_hash = hash;
+		}
+
+		// insert a fork at block 2, which becomes best block
+		let best_hash = insert_block(
+			&backend,
+			1,
+			blocks[0],
+			None,
+			sp_core::H256::random(),
+			vec![42.into()],
+			None
+		);
+		assert!(backend.remove_leaf_block(&best_hash).is_err());
+		assert!(backend.have_state_at(&prev_hash, 1));
+		backend.remove_leaf_block(&prev_hash).unwrap();
+		assert_eq!(None, backend.blockchain().header(BlockId::hash(prev_hash.clone())).unwrap());
+		assert!(!backend.have_state_at(&prev_hash, 1));
+	}
 }

client/finality-grandpa/src/communication/tests.rs

@@ -295,6 +295,7 @@ fn good_commit_leads_to_relay() {
 					let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened {
 						remote: sender_id.clone(),
 						protocol: GRANDPA_PROTOCOL_NAME.into(),
+						negotiated_fallback: None,
 						role: ObservedRole::Full,
 					});
 
@@ -308,6 +309,7 @@ fn good_commit_leads_to_relay() {
 					let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened {
 						remote: receiver_id.clone(),
 						protocol: GRANDPA_PROTOCOL_NAME.into(),
+						negotiated_fallback: None,
 						role: ObservedRole::Full,
 					});
 
@@ -442,6 +444,7 @@ fn bad_commit_leads_to_report() {
 					let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened {
 						remote: sender_id.clone(),
 						protocol: GRANDPA_PROTOCOL_NAME.into(),
+						negotiated_fallback: None,
 						role: ObservedRole::Full,
 					});
 					let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived {

client/finality-grandpa/src/lib.rs

@@ -690,6 +690,7 @@ pub struct GrandpaParams<Block: BlockT, C, N, SC, VR> {
 pub fn grandpa_peers_set_config() -> sc_network::config::NonDefaultSetConfig {
 	sc_network::config::NonDefaultSetConfig {
 		notifications_protocol: communication::GRANDPA_PROTOCOL_NAME.into(),
+		fallback_names: Vec::new(),
 		// Notifications reach ~256kiB in size at the time of writing on Kusama and Polkadot.
 		max_notification_size: 1024 * 1024,
 		set_config: sc_network::config::SetConfig {
@@ -1134,12 +1135,12 @@ fn local_authority_id(
 	voters: &VoterSet<AuthorityId>,
 	keystore: Option<&SyncCryptoStorePtr>,
 ) -> Option<AuthorityId> {
-	keystore.and_then(|keystore| { 
+	keystore.and_then(|keystore| {
 		voters
 		.iter()
 		.find(|(p, _)| {
 			SyncCryptoStore::has_keys(&**keystore, &[(p.to_raw_vec(), AuthorityId::ID)])
 		})
-		.map(|(p, _)| p.clone()) 
+		.map(|(p, _)| p.clone())
 	})
 }

client/light/src/backend.rs

@@ -246,6 +246,13 @@ impl<S, Block> ClientBackend<Block> for Backend<S, HashFor<Block>>
 		Err(ClientError::NotAvailableOnLightClient)
 	}
 
+	fn remove_leaf_block(
+		&self,
+		_hash: &Block::Hash,
+	) -> ClientResult<()> {
+		Err(ClientError::NotAvailableOnLightClient)
+	}
+
 	fn get_import_lock(&self) -> &RwLock<()> {
 		&self.import_lock
 	}

client/network-gossip/src/bridge.rs

@@ -188,7 +188,7 @@ impl<B: BlockT> Future for GossipEngine<B> {
 							Event::SyncDisconnected { remote } => {
 								this.network.remove_set_reserved(remote, this.protocol.clone());
 							}
-							Event::NotificationStreamOpened { remote, protocol, role } => {
+							Event::NotificationStreamOpened { remote, protocol, role, .. } => {
 								if protocol != this.protocol {
 									continue;
 								}
@@ -416,6 +416,7 @@ mod tests {
 			Event::NotificationStreamOpened {
 				remote: remote_peer.clone(),
 				protocol: protocol.clone(),
+				negotiated_fallback: None,
 				role: ObservedRole::Authority,
 			}
 		).expect("Event stream is unbounded; qed.");
@@ -575,6 +576,7 @@ mod tests {
 				Event::NotificationStreamOpened {
 					remote: remote_peer.clone(),
 					protocol: protocol.clone(),
+					negotiated_fallback: None,
 					role: ObservedRole::Authority,
 				}
 			).expect("Event stream is unbounded; qed.");

client/network/src/behaviour.rs

@@ -124,6 +124,11 @@ pub enum BehaviourOut<B: BlockT> {
 		remote: PeerId,
 		/// The concerned protocol. Each protocol uses a different substream.
 		protocol: Cow<'static, str>,
+		/// If the negotiation didn't use the main name of the protocol (the one in
+		/// `notifications_protocol`), then this field contains which name has actually been
+		/// used.
+		/// See also [`crate::Event::NotificationStreamOpened`].
+		negotiated_fallback: Option<Cow<'static, str>>,
 		/// Object that permits sending notifications to the peer.
 		notifications_sink: NotificationsSink,
 		/// Role of the remote.
@@ -324,10 +329,13 @@ Behaviour<B> {
 					&target, &self.block_request_protocol_name, buf, pending_response, IfDisconnected::ImmediateError,
 				);
 			},
-			CustomMessageOutcome::NotificationStreamOpened { remote, protocol, roles, notifications_sink } => {
+			CustomMessageOutcome::NotificationStreamOpened {
+				remote, protocol, negotiated_fallback, roles, notifications_sink
+			} => {
 				self.events.push_back(BehaviourOut::NotificationStreamOpened {
 					remote,
 					protocol,
+					negotiated_fallback,
 					role: reported_roles_to_observed_role(roles),
 					notifications_sink: notifications_sink.clone(),
 				});

client/network/src/config.rs

@@ -541,6 +541,13 @@ pub struct NonDefaultSetConfig {
 	/// > **Note**: This field isn't present for the default set, as this is handled internally
 	/// >           by the networking code.
 	pub notifications_protocol: Cow<'static, str>,
+	/// If the remote reports that it doesn't support the protocol indicated in the
+	/// `notifications_protocol` field, then each of these fallback names will be tried one by
+	/// one.
+	///
+	/// If a fallback is used, it will be reported in
+	/// [`crate::Event::NotificationStreamOpened::negotiated_fallback`].
+	pub fallback_names: Vec<Cow<'static, str>>,
 	/// Maximum allowed size of single notifications.
 	pub max_notification_size: u64,
 	/// Base configuration.
@@ -553,6 +560,7 @@ impl NonDefaultSetConfig {
 		NonDefaultSetConfig {
 			notifications_protocol,
 			max_notification_size,
+			fallback_names: Vec::new(),
 			set_config: SetConfig {
 				in_peers: 0,
 				out_peers: 0,

client/network/src/gossip/tests.rs

@@ -159,6 +159,7 @@ fn build_nodes_one_proto()
 		extra_sets: vec![
 			config::NonDefaultSetConfig {
 				notifications_protocol: PROTOCOL_NAME,
+				fallback_names: Vec::new(),
 				max_notification_size: 1024 * 1024,
 				set_config: Default::default()
 			}
@@ -173,6 +174,7 @@ fn build_nodes_one_proto()
 		extra_sets: vec![
 			config::NonDefaultSetConfig {
 				notifications_protocol: PROTOCOL_NAME,
+				fallback_names: Vec::new(),
 				max_notification_size: 1024 * 1024,
 				set_config: config::SetConfig {
 					reserved_nodes: vec![config::MultiaddrWithPeerId {