-
Notifications
You must be signed in to change notification settings - Fork 258
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
lightclient: Add support for multi-chain usecase #1238
Changes from 37 commits
4cdfe7b
962aeff
3fd6929
220bff3
b50e53d
4655a65
f91b10e
0616aa9
c2dba83
1876b42
6d945f5
b16c5f3
7ef743d
659ea8b
8abe5aa
d873f94
d095a51
c70f655
e2b9def
acaf279
e164b85
ce56fd4
7abaf88
1bafe50
93e1260
0efba8f
bcbde26
d9bd705
c0be204
79dfbd1
43a1c77
f7ef096
2f8dd47
0ee52e6
da4ee7e
deb648e
2f82e91
46a6adc
15811fc
56ca927
0cfe8d3
25581e4
84bf673
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,8 @@ | |
// This file is dual-licensed as Apache-2.0 or GPL-3.0. | ||
// see LICENSE for license details. | ||
|
||
use std::iter; | ||
|
||
use super::{ | ||
background::{BackgroundTask, FromSubxt, MethodResponse}, | ||
LightClientRpcError, | ||
|
@@ -11,14 +13,38 @@ use tokio::sync::{mpsc, mpsc::error::SendError, oneshot}; | |
|
||
use super::platform::build_platform; | ||
|
||
pub const LOG_TARGET: &str = "light-client"; | ||
pub const LOG_TARGET: &str = "subxt-light-client"; | ||
|
||
/// A raw light-client RPC implementation that can connect to multiple chains. | ||
#[derive(Clone)] | ||
pub struct RawLightClientRpc { | ||
/// Communicate with the backend task that multiplexes the responses | ||
/// back to the frontend. | ||
to_backend: mpsc::UnboundedSender<FromSubxt>, | ||
} | ||
|
||
impl RawLightClientRpc { | ||
/// Construct a [`LightClientRpc`] that can communicated with the provided chain. | ||
/// | ||
/// # Note | ||
/// | ||
/// This uses the same underlying instance created by [`LightClientRpc::new_from_client`]. | ||
pub fn for_chain(&self, chain_id: smoldot_light::ChainId) -> LightClientRpc { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I found the naming of this a bit odd, I would prefer There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Another question is it possible to connect the same chain more than once? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep, its possible to use the same chain ID to create multiple instances of the light client There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Have tested this locally with the following patch, because I was also curious if there's anything else to be done here: let similar_chain: LightClient<PolkadotConfig> =
raw_light_client.for_chain(parachain_chain_id).await?;
let mut sub = similar_chain.blocks().subscribe_all().await?;
tokio::spawn(async move {
while let Some(value) = sub.next().await {
let value = value.expect("OPS");
println!(" Background value {:?}", value.hash());
}
}); This will subscribe to all the blocks of the asset-hub and indeed we get blocks a bit earlier than our There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I guess I like |
||
LightClientRpc { | ||
to_backend: self.to_backend.clone(), | ||
chain_id, | ||
} | ||
} | ||
} | ||
|
||
/// The light-client RPC implementation that is used to connect with the chain. | ||
#[derive(Clone)] | ||
pub struct LightClientRpc { | ||
/// Communicate with the backend task that multiplexes the responses | ||
/// back to the frontend. | ||
to_backend: mpsc::UnboundedSender<FromSubxt>, | ||
/// The chain ID to target for requests. | ||
chain_id: smoldot_light::ChainId, | ||
} | ||
|
||
impl LightClientRpc { | ||
|
@@ -31,7 +57,16 @@ impl LightClientRpc { | |
/// | ||
/// ## Panics | ||
/// | ||
/// Panics if being called outside of `tokio` runtime context. | ||
/// The panic behaviour depends on the feature flag being used: | ||
/// | ||
/// ### Native | ||
/// | ||
/// Panics when called outside of a `tokio` runtime context. | ||
/// | ||
/// ### Web | ||
/// | ||
/// If smoldot panics, then the promise created will be leaked. For more details, see | ||
/// https://docs.rs/wasm-bindgen-futures/latest/wasm_bindgen_futures/fn.future_to_promise.html. | ||
pub fn new( | ||
config: smoldot_light::AddChainConfig<'_, (), impl Iterator<Item = smoldot_light::ChainId>>, | ||
lexnv marked this conversation as resolved.
Show resolved
Hide resolved
|
||
) -> Result<LightClientRpc, LightClientRpcError> { | ||
|
@@ -46,22 +81,61 @@ impl LightClientRpc { | |
.add_chain(config) | ||
.map_err(|err| LightClientRpcError::AddChainError(err.to_string()))?; | ||
|
||
let (to_backend, backend) = mpsc::unbounded_channel(); | ||
|
||
// `json_rpc_responses` can only be `None` if we had passed `json_rpc: Disabled`. | ||
let rpc_responses = json_rpc_responses.expect("Light client RPC configured; qed"); | ||
|
||
let raw_client = Self::new_from_client( | ||
client, | ||
iter::once(AddedChain { | ||
chain_id, | ||
rpc_responses, | ||
}), | ||
); | ||
Ok(raw_client.for_chain(chain_id)) | ||
} | ||
|
||
/// Constructs a new [`RawLightClientRpc`] from the raw smoldot client. | ||
/// | ||
/// Receives a list of RPC objects as a result of calling `smoldot_light::Client::add_chain`. | ||
/// This [`RawLightClientRpc`] can target different chains using [`RawLightClientRpc::for_chain`] method. | ||
/// | ||
/// ## Panics | ||
/// | ||
/// The panic behaviour depends on the feature flag being used: | ||
/// | ||
/// ### Native | ||
/// | ||
/// Panics when called outside of a `tokio` runtime context. | ||
/// | ||
/// ### Web | ||
/// | ||
/// If smoldot panics, then the promise created will be leaked. For more details, see | ||
/// https://docs.rs/wasm-bindgen-futures/latest/wasm_bindgen_futures/fn.future_to_promise.html. | ||
pub fn new_from_client<TPlat>( | ||
client: smoldot_light::Client<TPlat>, | ||
chains: impl Iterator<Item = AddedChain>, | ||
lexnv marked this conversation as resolved.
Show resolved
Hide resolved
|
||
) -> RawLightClientRpc | ||
where | ||
TPlat: smoldot_light::platform::PlatformRef + Clone, | ||
{ | ||
let (to_backend, backend) = mpsc::unbounded_channel(); | ||
let chains = chains.collect(); | ||
|
||
let future = async move { | ||
let mut task = BackgroundTask::new(client, chain_id); | ||
task.start_task(backend, rpc_responses).await; | ||
let mut task = BackgroundTask::new(client); | ||
task.start_task(backend, chains).await; | ||
}; | ||
|
||
#[cfg(feature = "native")] | ||
tokio::spawn(future); | ||
#[cfg(feature = "web")] | ||
wasm_bindgen_futures::spawn_local(future); | ||
|
||
Ok(LightClientRpc { to_backend }) | ||
RawLightClientRpc { to_backend } | ||
} | ||
|
||
/// Returns the chain ID of the current light-client. | ||
pub fn chain_id(&self) -> smoldot_light::ChainId { | ||
self.chain_id | ||
} | ||
|
||
/// Submits an RPC method request to the light-client. | ||
|
@@ -79,6 +153,7 @@ impl LightClientRpc { | |
method, | ||
params, | ||
sender, | ||
chain_id: self.chain_id, | ||
})?; | ||
|
||
Ok(receiver) | ||
|
@@ -107,8 +182,17 @@ impl LightClientRpc { | |
params, | ||
sub_id, | ||
sender, | ||
chain_id: self.chain_id, | ||
})?; | ||
|
||
Ok((sub_id_rx, receiver)) | ||
} | ||
} | ||
|
||
/// The added chain of the light-client. | ||
pub struct AddedChain { | ||
/// The id of the chain. | ||
pub chain_id: smoldot_light::ChainId, | ||
/// Producer of RPC responses for the chain. | ||
pub rpc_responses: smoldot_light::JsonRpcResponses, | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -33,8 +33,13 @@ mod platform; | |
#[allow(unused_imports)] | ||
pub use getrandom as _; | ||
|
||
pub use client::LightClientRpc; | ||
pub use smoldot_light::{AddChainConfig, AddChainConfigJsonRpc, ChainId}; | ||
pub use client::{AddedChain, LightClientRpc, RawLightClientRpc}; | ||
pub use smoldot_light::{ | ||
platform::PlatformRef, AddChainConfig, AddChainConfigJsonRpc, ChainId, Client, JsonRpcResponses, | ||
}; | ||
|
||
#[cfg(feature = "native")] | ||
pub use smoldot_light::platform::default::DefaultPlatform; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm generally thinking if it's better to expose all of the I think the ideal place to be here is:
So maybe here we could expose all of the smoldot stuff via a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep that makes sense to have different exports. I modified the exports as:
|
||
|
||
/// Light client error. | ||
#[derive(Debug, thiserror::Error)] | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,100 @@ | ||
use futures::StreamExt; | ||
use std::{iter, num::NonZeroU32}; | ||
use subxt::{ | ||
client::{LightClient, RawLightClient}, | ||
PolkadotConfig, | ||
}; | ||
|
||
// Generate an interface that we can use from the node's metadata. | ||
#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata_small.scale")] | ||
pub mod polkadot {} | ||
|
||
const POLKADOT_SPEC: &str = include_str!("../../artifacts/demo_chain_specs/polkadot.json"); | ||
const ASSET_HUB_SPEC: &str = | ||
include_str!("../../artifacts/demo_chain_specs/polkadot_asset_hub.json"); | ||
|
||
#[tokio::main] | ||
async fn main() -> Result<(), Box<dyn std::error::Error>> { | ||
// The smoldot logs are informative: | ||
tracing_subscriber::fmt::init(); | ||
|
||
// Connecting to a parachain is a multi step process. | ||
|
||
// Step 1. Construct a new smoldot client. | ||
let mut client = subxt_lightclient::Client::new(subxt_lightclient::DefaultPlatform::new( | ||
"subxt-example-light-client".into(), | ||
"version-0".into(), | ||
)); | ||
|
||
// Step 2. Connect to the relay chain of the parachain. For this example, the Polkadot relay chain. | ||
let polkadot_connection = client | ||
.add_chain(subxt_lightclient::AddChainConfig { | ||
specification: POLKADOT_SPEC, | ||
json_rpc: subxt_lightclient::AddChainConfigJsonRpc::Enabled { | ||
max_pending_requests: NonZeroU32::new(128).unwrap(), | ||
max_subscriptions: 1024, | ||
}, | ||
potential_relay_chains: iter::empty(), | ||
database_content: "", | ||
user_data: (), | ||
}) | ||
.expect("Light client chain added with valid spec; qed"); | ||
let polkadot_json_rpc_responses = polkadot_connection | ||
.json_rpc_responses | ||
.expect("Light client configured with json rpc enabled; qed"); | ||
let polkadot_chain_id = polkadot_connection.chain_id; | ||
|
||
// Step 3. Connect to the parachain. For this example, the Asset hub parachain. | ||
let assethub_connection = client | ||
.add_chain(subxt_lightclient::AddChainConfig { | ||
specification: ASSET_HUB_SPEC, | ||
json_rpc: subxt_lightclient::AddChainConfigJsonRpc::Enabled { | ||
max_pending_requests: NonZeroU32::new(128).unwrap(), | ||
max_subscriptions: 1024, | ||
}, | ||
// The chain specification of the asset hub parachain mentions that the identifier | ||
// of its relay chain is `polkadot`. | ||
potential_relay_chains: [polkadot_chain_id].into_iter(), | ||
database_content: "", | ||
user_data: (), | ||
}) | ||
.expect("Light client chain added with valid spec; qed"); | ||
let parachain_json_rpc_responses = assethub_connection | ||
.json_rpc_responses | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is it possible to write a wrapper for this API? It's quite annoying to do this unwrap because the RPC client should always be enabled in smoldot? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep, I was thinking of creating a bit of a higher level API as a followup to this. Ideally we could provide:
let api = LightClient::<PolkadotConfig>::builder()
.bootnodes([
"/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
])
.build_from_url("ws://127.0.0.1:9944")
.await?;
/// Naming tbd, suggesting that's somewhere in the middle between super high level and super low level API.
let client: MediumLightClient = MediumLightClient::builder()
.add_chain(
AddChianBuilder::with_identifier( "polkadot" )
.with_spec( include_str!( "../assets/polkadot_chain_spec.json")
)
.add_chain(
AddChainBuilder::with_identifier( "asset-hub")
.relay_chain( "polkadot")
.from_url("ws://127.0.0.1:9944")
)
.build().await;
let polkadot_api = client.for_chain( "polkadot"); Something like that, but I didn't put a proper thought into the API. We still need a way to identify the chains while adding them to the client. And another approach here, would be to have a Another thing worth exploring would be extending the Still believe having the ability for users to turn a raw light client into something that subxt can use under the scene would cover the most complex use-cases :D There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yup, I'm generally for the idea too of having:
But yeah it does need some thought and experimentation to find the ideal interface for this stuff I think; we shouldn't rush into it :) |
||
.expect("Light client configured with json rpc enabled; qed"); | ||
let parachain_chain_id = assethub_connection.chain_id; | ||
|
||
// Step 4. Turn the smoldot client into a raw client. | ||
let raw_light_client = RawLightClient::builder() | ||
.add_chain(polkadot_chain_id, polkadot_json_rpc_responses) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. hmm, so this a little complicated to first call "add_chain" and then after call "for_chain". I don't follow why we can't hide that from this API and just provide "connect_to_chain(chain_id, rpc_stream)" to avoid having to functions to call. Perhaps you need pass in the client as well There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The It might be possible to introduce another subxt frontend -> subxt backend to propagate this to the smoldot client from the background task and add the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's fine for now but perhaps worth writing up an issue to look into. My point is initializing and using the API is a bit complicated but There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep that's a good point, raised #1253 to handle it later, thanks! |
||
.add_chain(parachain_chain_id, parachain_json_rpc_responses) | ||
.build(client) | ||
.await?; | ||
|
||
// Step 5. Obtain a client to target the relay chain and the parachain. | ||
let polkadot_api: LightClient<PolkadotConfig> = | ||
lexnv marked this conversation as resolved.
Show resolved
Hide resolved
|
||
raw_light_client.for_chain(polkadot_chain_id).await?; | ||
let parachain_api: LightClient<PolkadotConfig> = | ||
raw_light_client.for_chain(parachain_chain_id).await?; | ||
|
||
// Step 6. Subscribe to the finalized blocks of the chains. | ||
let polkadot_sub = polkadot_api | ||
.blocks() | ||
.subscribe_finalized() | ||
.await? | ||
.map(|block| ("Polkadot", block)); | ||
let parachain_sub = parachain_api | ||
.blocks() | ||
.subscribe_finalized() | ||
.await? | ||
.map(|block| ("AssetHub", block)); | ||
let mut stream_combinator = futures::stream::select(polkadot_sub, parachain_sub); | ||
|
||
while let Some((chain, block)) = stream_combinator.next().await { | ||
let block = block?; | ||
|
||
println!(" Chain {:?} hash={:?}", chain, block.hash()); | ||
} | ||
|
||
Ok(()) | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I wonder if it's possible to obtain a more condensed version of this file; there's one here for polkadot which is much smaller:
https://github.com/paritytech/substrate-connect/blob/main/projects/extension/assets/chainspecs/polkadot.json
It'd be great to know how to programmatically get hold of the smallest ones possible!
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I have created this issue that I ll tackle shortly 👍