Merge remote-tracking branch 'origin/master' into kiz-fix-pools-doc

.gitlab-ci.yml

@@ -48,9 +48,9 @@ variables:
   CARGO_INCREMENTAL: 0
   DOCKER_OS: "debian:stretch"
   ARCH: "x86_64"
-  CI_IMAGE: "paritytech/ci-linux@sha256:a8b63e4f1ab37f90034b9edd3a936a1d4ae897560e25010e0f2fccd8274c6f27" # staging 2023-03-28
+  CI_IMAGE: "paritytech/ci-linux:production"
   BUILDAH_IMAGE: "quay.io/buildah/stable:v1.27"
-  RELENG_SCRIPTS_BRANCH: "mira/pg-13" # TODO: back to master when the ci image is moved back to prod
+  RELENG_SCRIPTS_BRANCH: "master"
   RUSTY_CACHIER_SINGLE_BRANCH: master
   RUSTY_CACHIER_DONT_OPERATE_ON_MAIN_BRANCH: "true"
   RUSTY_CACHIER_COMPRESSION_METHOD: zstd

bin/node/runtime/Cargo.toml

@@ -46,7 +46,7 @@ sp-io = { version = "7.0.0", default-features = false, path = "../../../primitiv
 frame-executive = { version = "4.0.0-dev", default-features = false, path = "../../../frame/executive" }
 frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../../../frame/benchmarking" }
 frame-benchmarking-pallet-pov = { version = "4.0.0-dev", default-features = false, path = "../../../frame/benchmarking/pov" }
-frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../frame/support" }
+frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../frame/support", features = ["tuples-96"] }
 frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system" }
 frame-system-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system/benchmarking", optional = true }
 frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../../../frame/election-provider-support" }

client/keystore/src/local.rs

@@ -63,21 +63,50 @@ impl LocalKeystore {
 	) -> Result<Option<Pair>> {
 		self.0.read().key_pair::<Pair>(public)
 	}
-}
 
-impl Keystore for LocalKeystore {
-	fn sr25519_public_keys(&self, key_type: KeyTypeId) -> Vec<sr25519::Public> {
+	fn public_keys<T: CorePair>(&self, key_type: KeyTypeId) -> Vec<T::Public> {
 		self.0
 			.read()
 			.raw_public_keys(key_type)
 			.map(|v| {
-				v.into_iter()
-					.filter_map(|k| sr25519::Public::from_slice(k.as_slice()).ok())
-					.collect()
+				v.into_iter().filter_map(|k| T::Public::from_slice(k.as_slice()).ok()).collect()
 			})
 			.unwrap_or_default()
 	}
 
+	fn generate_new<T: CorePair>(
+		&self,
+		key_type: KeyTypeId,
+		seed: Option<&str>,
+	) -> std::result::Result<T::Public, TraitError> {
+		let pair = match seed {
+			Some(seed) => self.0.write().insert_ephemeral_from_seed_by_type::<T>(seed, key_type),
+			None => self.0.write().generate_by_type::<T>(key_type),
+		}
+		.map_err(|e| -> TraitError { e.into() })?;
+		Ok(pair.public())
+	}
+
+	fn sign<T: CorePair>(
+		&self,
+		key_type: KeyTypeId,
+		public: &T::Public,
+		msg: &[u8],
+	) -> std::result::Result<Option<T::Signature>, TraitError> {
+		let signature = self
+			.0
+			.read()
+			.key_pair_by_type::<T>(public, key_type)?
+			.map(|pair| pair.sign(msg));
+		Ok(signature)
+	}
+}
+
+impl Keystore for LocalKeystore {
+	fn sr25519_public_keys(&self, key_type: KeyTypeId) -> Vec<sr25519::Public> {
+		self.public_keys::<sr25519::Pair>(key_type)
+	}
+
 	/// Generate a new pair compatible with the 'ed25519' signature scheme.
 	///
 	/// If the `[seed]` is `Some` then the key will be ephemeral and stored in memory.
@@ -86,16 +115,7 @@ impl Keystore for LocalKeystore {
 		key_type: KeyTypeId,
 		seed: Option<&str>,
 	) -> std::result::Result<sr25519::Public, TraitError> {
-		let pair = match seed {
-			Some(seed) => self
-				.0
-				.write()
-				.insert_ephemeral_from_seed_by_type::<sr25519::Pair>(seed, key_type),
-			None => self.0.write().generate_by_type::<sr25519::Pair>(key_type),
-		}
-		.map_err(|e| -> TraitError { e.into() })?;
-
-		Ok(pair.public())
+		self.generate_new::<sr25519::Pair>(key_type, seed)
 	}
 
 	fn sr25519_sign(
@@ -104,12 +124,7 @@ impl Keystore for LocalKeystore {
 		public: &sr25519::Public,
 		msg: &[u8],
 	) -> std::result::Result<Option<sr25519::Signature>, TraitError> {
-		let res = self
-			.0
-			.read()
-			.key_pair_by_type::<sr25519::Pair>(public, key_type)?
-			.map(|pair| pair.sign(msg));
-		Ok(res)
+		self.sign::<sr25519::Pair>(key_type, public, msg)
 	}
 
 	fn sr25519_vrf_sign(
@@ -118,24 +133,16 @@ impl Keystore for LocalKeystore {
 		public: &sr25519::Public,
 		transcript_data: VRFTranscriptData,
 	) -> std::result::Result<Option<VRFSignature>, TraitError> {
-		let res = self.0.read().key_pair_by_type::<sr25519::Pair>(public, key_type)?.map(|pair| {
+		let sig = self.0.read().key_pair_by_type::<sr25519::Pair>(public, key_type)?.map(|pair| {
 			let transcript = make_transcript(transcript_data);
 			let (inout, proof, _) = pair.as_ref().vrf_sign(transcript);
 			VRFSignature { output: inout.to_output(), proof }
 		});
-		Ok(res)
+		Ok(sig)
 	}
 
 	fn ed25519_public_keys(&self, key_type: KeyTypeId) -> Vec<ed25519::Public> {
-		self.0
-			.read()
-			.raw_public_keys(key_type)
-			.map(|v| {
-				v.into_iter()
-					.filter_map(|k| ed25519::Public::from_slice(k.as_slice()).ok())
-					.collect()
-			})
-			.unwrap_or_default()
+		self.public_keys::<ed25519::Pair>(key_type)
 	}
 
 	/// Generate a new pair compatible with the 'sr25519' signature scheme.
@@ -146,16 +153,7 @@ impl Keystore for LocalKeystore {
 		key_type: KeyTypeId,
 		seed: Option<&str>,
 	) -> std::result::Result<ed25519::Public, TraitError> {
-		let pair = match seed {
-			Some(seed) => self
-				.0
-				.write()
-				.insert_ephemeral_from_seed_by_type::<ed25519::Pair>(seed, key_type),
-			None => self.0.write().generate_by_type::<ed25519::Pair>(key_type),
-		}
-		.map_err(|e| -> TraitError { e.into() })?;
-
-		Ok(pair.public())
+		self.generate_new::<ed25519::Pair>(key_type, seed)
 	}
 
 	fn ed25519_sign(
@@ -164,24 +162,11 @@ impl Keystore for LocalKeystore {
 		public: &ed25519::Public,
 		msg: &[u8],
 	) -> std::result::Result<Option<ed25519::Signature>, TraitError> {
-		let res = self
-			.0
-			.read()
-			.key_pair_by_type::<ed25519::Pair>(public, key_type)?
-			.map(|pair| pair.sign(msg));
-		Ok(res)
+		self.sign::<ed25519::Pair>(key_type, public, msg)
 	}
 
 	fn ecdsa_public_keys(&self, key_type: KeyTypeId) -> Vec<ecdsa::Public> {
-		self.0
-			.read()
-			.raw_public_keys(key_type)
-			.map(|v| {
-				v.into_iter()
-					.filter_map(|k| ecdsa::Public::from_slice(k.as_slice()).ok())
-					.collect()
-			})
-			.unwrap_or_default()
+		self.public_keys::<ecdsa::Pair>(key_type)
 	}
 
 	/// Generate a new pair compatible with the 'ecdsa' signature scheme.
@@ -192,14 +177,7 @@ impl Keystore for LocalKeystore {
 		key_type: KeyTypeId,
 		seed: Option<&str>,
 	) -> std::result::Result<ecdsa::Public, TraitError> {
-		let pair = match seed {
-			Some(seed) =>
-				self.0.write().insert_ephemeral_from_seed_by_type::<ecdsa::Pair>(seed, key_type),
-			None => self.0.write().generate_by_type::<ecdsa::Pair>(key_type),
-		}
-		.map_err(|e| -> TraitError { e.into() })?;
-
-		Ok(pair.public())
+		self.generate_new::<ecdsa::Pair>(key_type, seed)
 	}
 
 	fn ecdsa_sign(
@@ -208,12 +186,7 @@ impl Keystore for LocalKeystore {
 		public: &ecdsa::Public,
 		msg: &[u8],
 	) -> std::result::Result<Option<ecdsa::Signature>, TraitError> {
-		let res = self
-			.0
-			.read()
-			.key_pair_by_type::<ecdsa::Pair>(public, key_type)?
-			.map(|pair| pair.sign(msg));
-		Ok(res)
+		self.sign::<ecdsa::Pair>(key_type, public, msg)
 	}
 
 	fn ecdsa_sign_prehashed(
@@ -222,12 +195,12 @@ impl Keystore for LocalKeystore {
 		public: &ecdsa::Public,
 		msg: &[u8; 32],
 	) -> std::result::Result<Option<ecdsa::Signature>, TraitError> {
-		let res = self
+		let sig = self
 			.0
 			.read()
 			.key_pair_by_type::<ecdsa::Pair>(public, key_type)?
 			.map(|pair| pair.sign_prehashed(msg));
-		Ok(res)
+		Ok(sig)
 	}
 
 	fn insert(

client/network/src/config.rs

@@ -22,6 +22,7 @@
 //! See the documentation of [`Params`].
 
 pub use crate::{
+	protocol::NotificationsSink,
 	request_responses::{
 		IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig,
 	},
@@ -31,7 +32,12 @@ pub use crate::{
 use codec::Encode;
 use libp2p::{identity::Keypair, multiaddr, Multiaddr, PeerId};
 use prometheus_endpoint::Registry;
-pub use sc_network_common::{role::Role, sync::warp::WarpSyncProvider, ExHashT};
+pub use sc_network_common::{
+	role::{Role, Roles},
+	sync::warp::WarpSyncProvider,
+	ExHashT,
+};
+use sc_utils::mpsc::TracingUnboundedSender;
 use zeroize::Zeroize;
 
 use sp_runtime::traits::Block as BlockT;
@@ -714,6 +720,9 @@ pub struct Params<Block: BlockT> {
 	/// Block announce protocol configuration
 	pub block_announce_config: NonDefaultSetConfig,
 
+	/// TX channel for direct communication with `SyncingEngine` and `Protocol`.
+	pub tx: TracingUnboundedSender<crate::event::SyncEvent<Block>>,
+
 	/// Request response protocol configurations
 	pub request_response_protocol_configs: Vec<RequestResponseConfig>,
 }

client/network/src/event.rs

@@ -19,12 +19,14 @@
 //! Network event types. These are are not the part of the protocol, but rather
 //! events that happen on the network like DHT get/put results received.
 
-use crate::types::ProtocolName;
+use crate::{types::ProtocolName, NotificationsSink};
 
 use bytes::Bytes;
+use futures::channel::oneshot;
 use libp2p::{core::PeerId, kad::record::Key};
 
-use sc_network_common::role::ObservedRole;
+use sc_network_common::{role::ObservedRole, sync::message::BlockAnnouncesHandshake};
+use sp_runtime::traits::Block as BlockT;
 
 /// Events generated by DHT as a response to get_value and put_value requests.
 #[derive(Debug, Clone)]
@@ -90,3 +92,44 @@ pub enum Event {
 		messages: Vec<(ProtocolName, Bytes)>,
 	},
 }
+
+/// Event sent to `SyncingEngine`
+// TODO: remove once `NotificationService` is implemented.
+pub enum SyncEvent<B: BlockT> {
+	/// Opened a substream with the given node with the given notifications protocol.
+	///
+	/// The protocol is always one of the notification protocols that have been registered.
+	NotificationStreamOpened {
+		/// Node we opened the substream with.
+		remote: PeerId,
+		/// Received handshake.
+		received_handshake: BlockAnnouncesHandshake<B>,
+		/// Notification sink.
+		sink: NotificationsSink,
+		/// Channel for reporting accept/reject of the substream.
+		tx: oneshot::Sender<bool>,
+	},
+
+	/// Closed a substream with the given node. Always matches a corresponding previous
+	/// `NotificationStreamOpened` message.
+	NotificationStreamClosed {
+		/// Node we closed the substream with.
+		remote: PeerId,
+	},
+
+	/// Notification sink was replaced.
+	NotificationSinkReplaced {
+		/// Node we closed the substream with.
+		remote: PeerId,
+		/// Notification sink.
+		sink: NotificationsSink,
+	},
+
+	/// Received one or more messages from the given node using the given protocol.
+	NotificationsReceived {
+		/// Node we received the message from.
+		remote: PeerId,
+		/// Concerned protocol and associated message.
+		messages: Vec<Bytes>,
+	},
+}

client/network/src/lib.rs

@@ -259,7 +259,7 @@ pub mod request_responses;
 pub mod types;
 pub mod utils;
 
-pub use event::{DhtEvent, Event};
+pub use event::{DhtEvent, Event, SyncEvent};
 #[doc(inline)]
 pub use libp2p::{multiaddr, Multiaddr, PeerId};
 pub use request_responses::{IfDisconnected, RequestFailure, RequestResponseConfig};
@@ -278,8 +278,8 @@ pub use service::{
 		NetworkStatusProvider, NetworkSyncForkRequest, NotificationSender as NotificationSenderT,
 		NotificationSenderError, NotificationSenderReady,
 	},
-	DecodingError, Keypair, NetworkService, NetworkWorker, NotificationSender, OutboundFailure,
-	PublicKey,
+	DecodingError, Keypair, NetworkService, NetworkWorker, NotificationSender, NotificationsSink,
+	OutboundFailure, PublicKey,
 };
 pub use types::ProtocolName;