From 9159cf2845576d58fa7d2c09dde372db91662838 Mon Sep 17 00:00:00 2001 From: Luca Joss Date: Wed, 26 Jun 2024 15:26:22 +0200 Subject: [PATCH 1/9] Add /genesis_chunked query --- rpc/src/client.rs | 8 +++++ rpc/src/endpoint.rs | 1 + rpc/src/endpoint/genesis_chunked.rs | 52 +++++++++++++++++++++++++++++ rpc/src/method.rs | 4 +++ 4 files changed, 65 insertions(+) create mode 100644 rpc/src/endpoint/genesis_chunked.rs diff --git a/rpc/src/client.rs b/rpc/src/client.rs index b90472281..24454aefa 100644 --- a/rpc/src/client.rs +++ b/rpc/src/client.rs @@ -271,6 +271,14 @@ pub trait Client { Ok(self.perform(genesis::Request::default()).await?.genesis) } + /// `/genesis_chunked`: get genesis file in multiple chunks. + async fn genesis_chunked( + &self, + chunk: Option, + ) -> Result { + self.perform(genesis_chunked::Request::new(chunk)).await + } + /// `/net_info`: obtain information about P2P and other network connections. async fn net_info(&self) -> Result { self.perform(net_info::Request).await diff --git a/rpc/src/endpoint.rs b/rpc/src/endpoint.rs index f04c2a0df..bd8b79e3c 100644 --- a/rpc/src/endpoint.rs +++ b/rpc/src/endpoint.rs @@ -13,6 +13,7 @@ pub mod consensus_params; pub mod consensus_state; pub mod evidence; pub mod genesis; +pub mod genesis_chunked; pub mod header; pub mod header_by_hash; pub mod health; diff --git a/rpc/src/endpoint/genesis_chunked.rs b/rpc/src/endpoint/genesis_chunked.rs new file mode 100644 index 000000000..9bd22f25f --- /dev/null +++ b/rpc/src/endpoint/genesis_chunked.rs @@ -0,0 +1,52 @@ +//! `/genesis` endpoint JSON-RPC wrapper + +use alloc::{string::String, vec::Vec}; +use serde::{Deserialize, Serialize}; +use tendermint_proto::serializers; + +use crate::{dialect::Dialect, request::RequestMessage}; + +/// Get the genesis state in multiple chunks for the current chain +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct Request { + pub chunk: Option, +} + +impl Request { + pub fn new(chunk: Option) -> Self { + Self { chunk } + } +} + +impl RequestMessage for Request { + fn method(&self) -> crate::Method { + crate::Method::GenesisChunked + } +} + +impl crate::Request for Request +where + S: Dialect, +{ + type Response = Response; +} + +impl crate::SimpleRequest for Request +where + S: Dialect, +{ + type Output = Response; +} + +/// Block responses +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Response { + #[serde(with = "serializers::from_str")] + pub chunk: u64, + #[serde(with = "serializers::from_str")] + pub total: u64, + #[serde(with = "serializers::bytes::base64string")] + pub data: Vec, +} + +impl crate::Response for Response {} diff --git a/rpc/src/method.rs b/rpc/src/method.rs index 6f91c5d5f..98b18ffd0 100644 --- a/rpc/src/method.rs +++ b/rpc/src/method.rs @@ -56,6 +56,9 @@ pub enum Method { /// Get genesis file Genesis, + /// Get genesis in multiple chunks file + GenesisChunked, + /// Get block header Header, @@ -109,6 +112,7 @@ impl Method { Method::ConsensusParams => "consensus_params", Method::ConsensusState => "consensus_state", Method::Genesis => "genesis", + Method::GenesisChunked => "genesis_chunked", Method::Header => "header", Method::HeaderByHash => "header_by_hash", Method::Health => "health", From 35aa28ae60c986742bff62b2bc7c7187d5e97552 Mon Sep 17 00:00:00 2001 From: Luca Joss Date: Wed, 26 Jun 2024 15:38:09 +0200 Subject: [PATCH 2/9] Add changelog entry --- .changelog/unreleased/features/1438-genesis-chunked-endpoint.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .changelog/unreleased/features/1438-genesis-chunked-endpoint.md diff --git a/.changelog/unreleased/features/1438-genesis-chunked-endpoint.md b/.changelog/unreleased/features/1438-genesis-chunked-endpoint.md new file mode 100644 index 000000000..9933439f2 --- /dev/null +++ b/.changelog/unreleased/features/1438-genesis-chunked-endpoint.md @@ -0,0 +1,2 @@ +- `[tendermint-rpc]` Add `/genesis_chunked` RPC endpoint + ([\#1438](https://github.com/informalsystems/tendermint-rs/issues/1438)) \ No newline at end of file From f4eae0979198a68426f78df2e4f81067992309c4 Mon Sep 17 00:00:00 2001 From: Luca Joss Date: Wed, 26 Jun 2024 15:39:28 +0200 Subject: [PATCH 3/9] Fix comments --- rpc/src/endpoint/genesis_chunked.rs | 2 +- rpc/src/method.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/src/endpoint/genesis_chunked.rs b/rpc/src/endpoint/genesis_chunked.rs index 9bd22f25f..32bbcc266 100644 --- a/rpc/src/endpoint/genesis_chunked.rs +++ b/rpc/src/endpoint/genesis_chunked.rs @@ -1,4 +1,4 @@ -//! `/genesis` endpoint JSON-RPC wrapper +//! `/genesis_chunked` endpoint JSON-RPC wrapper use alloc::{string::String, vec::Vec}; use serde::{Deserialize, Serialize}; diff --git a/rpc/src/method.rs b/rpc/src/method.rs index 98b18ffd0..8614a2620 100644 --- a/rpc/src/method.rs +++ b/rpc/src/method.rs @@ -56,7 +56,7 @@ pub enum Method { /// Get genesis file Genesis, - /// Get genesis in multiple chunks file + /// Get genesis file in multiple chunks GenesisChunked, /// Get block header From 4a4afa2d26b4beb759b15570b26356f19ad543a7 Mon Sep 17 00:00:00 2001 From: Luca Joss Date: Thu, 27 Jun 2024 09:26:37 +0200 Subject: [PATCH 4/9] Update types of GenesisChunked request and response --- rpc/src/client.rs | 5 +---- rpc/src/endpoint/genesis_chunked.rs | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/rpc/src/client.rs b/rpc/src/client.rs index 24454aefa..84a71c8a0 100644 --- a/rpc/src/client.rs +++ b/rpc/src/client.rs @@ -272,10 +272,7 @@ pub trait Client { } /// `/genesis_chunked`: get genesis file in multiple chunks. - async fn genesis_chunked( - &self, - chunk: Option, - ) -> Result { + async fn genesis_chunked(&self, chunk: usize) -> Result { self.perform(genesis_chunked::Request::new(chunk)).await } diff --git a/rpc/src/endpoint/genesis_chunked.rs b/rpc/src/endpoint/genesis_chunked.rs index 32bbcc266..1e5cd20f9 100644 --- a/rpc/src/endpoint/genesis_chunked.rs +++ b/rpc/src/endpoint/genesis_chunked.rs @@ -1,6 +1,9 @@ //! `/genesis_chunked` endpoint JSON-RPC wrapper -use alloc::{string::String, vec::Vec}; +use alloc::{ + string::{String, ToString}, + vec::Vec, +}; use serde::{Deserialize, Serialize}; use tendermint_proto::serializers; @@ -9,12 +12,14 @@ use crate::{dialect::Dialect, request::RequestMessage}; /// Get the genesis state in multiple chunks for the current chain #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct Request { - pub chunk: Option, + pub chunk: String, } impl Request { - pub fn new(chunk: Option) -> Self { - Self { chunk } + pub fn new(chunk: usize) -> Self { + Self { + chunk: chunk.to_string(), + } } } @@ -41,10 +46,8 @@ where /// Block responses #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Response { - #[serde(with = "serializers::from_str")] - pub chunk: u64, - #[serde(with = "serializers::from_str")] - pub total: u64, + pub chunk: i32, + pub total: i32, #[serde(with = "serializers::bytes::base64string")] pub data: Vec, } From 56e837d9675841f40fd273004ccc4039d59d7181 Mon Sep 17 00:00:00 2001 From: Luca Joss Date: Thu, 27 Jun 2024 09:29:12 +0200 Subject: [PATCH 5/9] Convert chunk and total fields from GenesisChunked response --- rpc/src/endpoint/genesis_chunked.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rpc/src/endpoint/genesis_chunked.rs b/rpc/src/endpoint/genesis_chunked.rs index 1e5cd20f9..2678f005e 100644 --- a/rpc/src/endpoint/genesis_chunked.rs +++ b/rpc/src/endpoint/genesis_chunked.rs @@ -46,7 +46,9 @@ where /// Block responses #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Response { + #[serde(with = "serializers::from_str")] pub chunk: i32, + #[serde(with = "serializers::from_str")] pub total: i32, #[serde(with = "serializers::bytes::base64string")] pub data: Vec, From 52fbc035b91082873a0294baa8eea623d6f004d3 Mon Sep 17 00:00:00 2001 From: Luca Joss Date: Thu, 27 Jun 2024 15:08:35 +0200 Subject: [PATCH 6/9] Improve genesis_chunked_stream method --- rpc/src/client.rs | 28 ++++++++++++++++++++++++++-- rpc/src/endpoint/genesis_chunked.rs | 6 +++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/rpc/src/client.rs b/rpc/src/client.rs index 84a71c8a0..7a64df229 100644 --- a/rpc/src/client.rs +++ b/rpc/src/client.rs @@ -5,6 +5,7 @@ pub use compat::CompatMode; #[cfg(any(feature = "http-client", feature = "websocket-client"))] mod subscription; +use futures::Stream; #[cfg(any(feature = "http-client", feature = "websocket-client"))] pub use subscription::{Subscription, SubscriptionClient}; @@ -25,6 +26,7 @@ pub use transport::websocket::{ pub use transport::mock::{MockClient, MockRequestMatcher, MockRequestMethodMatcher}; use core::fmt; +use core::pin::Pin; use async_trait::async_trait; use serde::{de::DeserializeOwned, Serialize}; @@ -271,11 +273,33 @@ pub trait Client { Ok(self.perform(genesis::Request::default()).await?.genesis) } - /// `/genesis_chunked`: get genesis file in multiple chunks. - async fn genesis_chunked(&self, chunk: usize) -> Result { + async fn genesis_chunked(&self, chunk: u64) -> Result { self.perform(genesis_chunked::Request::new(chunk)).await } + /// `/genesis_chunked`: get genesis file in multiple chunks. + async fn genesis_chunked_stream( + &self, + ) -> Pin, Error>> + '_>> { + Box::pin(futures::stream::unfold(Some(0), move |chunk| async move { + // Verify if there are more chunks to fetch + let chunk = chunk?; + + match self.genesis_chunked(chunk).await { + Ok(response) => { + if response.chunk + 1 >= response.total { + // No more chunks to fetch + Some((Ok(response.data), None)) + } else { + // Emit this chunk and fetch the next chunk + Some((Ok(response.data), Some(response.chunk + 1))) + } + }, + Err(e) => Some((Err(e), None)), // Abort the stream + } + })) + } + /// `/net_info`: obtain information about P2P and other network connections. async fn net_info(&self) -> Result { self.perform(net_info::Request).await diff --git a/rpc/src/endpoint/genesis_chunked.rs b/rpc/src/endpoint/genesis_chunked.rs index 2678f005e..230b71be6 100644 --- a/rpc/src/endpoint/genesis_chunked.rs +++ b/rpc/src/endpoint/genesis_chunked.rs @@ -16,7 +16,7 @@ pub struct Request { } impl Request { - pub fn new(chunk: usize) -> Self { + pub fn new(chunk: u64) -> Self { Self { chunk: chunk.to_string(), } @@ -47,9 +47,9 @@ where #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Response { #[serde(with = "serializers::from_str")] - pub chunk: i32, + pub chunk: u64, #[serde(with = "serializers::from_str")] - pub total: i32, + pub total: u64, #[serde(with = "serializers::bytes::base64string")] pub data: Vec, } From d23bc9b8b49070a2b84b77e32d4a6e40918db689 Mon Sep 17 00:00:00 2001 From: Luca Joss Date: Thu, 27 Jun 2024 15:12:41 +0200 Subject: [PATCH 7/9] Add genesis-chunked CLI --- rpc/src/client/bin/main.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rpc/src/client/bin/main.rs b/rpc/src/client/bin/main.rs index d53de8602..7b15498cf 100644 --- a/rpc/src/client/bin/main.rs +++ b/rpc/src/client/bin/main.rs @@ -136,6 +136,8 @@ enum ClientRequest { ConsensusState, /// Get the node's genesis data. Genesis, + /// Get the node's genesis data by chunks + GenesisChunked, /// Get the node's health. Health, /// Request the latest block. @@ -413,6 +415,20 @@ where serde_json::to_string_pretty(&client.genesis::().await?) .map_err(Error::serde)? }, + ClientRequest::GenesisChunked => { + let mut data = Vec::new(); + let mut chunks = client.genesis_chunked_stream().await; + + while let Some(chunk) = chunks.next().await { + let mut chunk = chunk?; + data.append(&mut chunk); + } + + let genesis: tendermint::genesis::Genesis = + serde_json::from_slice(&data).map_err(Error::serde)?; + + serde_json::to_string_pretty(&genesis).map_err(Error::serde)? + }, ClientRequest::Health => { serde_json::to_string_pretty(&client.health().await?).map_err(Error::serde)? }, From 8fc0a1e776513891c0452ac3e28b9c6faa1377eb Mon Sep 17 00:00:00 2001 From: Luca Joss Date: Thu, 27 Jun 2024 15:27:51 +0200 Subject: [PATCH 8/9] Fix dependencies for genesis_chunked_stream --- rpc/src/client.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rpc/src/client.rs b/rpc/src/client.rs index 7a64df229..b073b47a8 100644 --- a/rpc/src/client.rs +++ b/rpc/src/client.rs @@ -5,7 +5,6 @@ pub use compat::CompatMode; #[cfg(any(feature = "http-client", feature = "websocket-client"))] mod subscription; -use futures::Stream; #[cfg(any(feature = "http-client", feature = "websocket-client"))] pub use subscription::{Subscription, SubscriptionClient}; @@ -26,7 +25,6 @@ pub use transport::websocket::{ pub use transport::mock::{MockClient, MockRequestMatcher, MockRequestMethodMatcher}; use core::fmt; -use core::pin::Pin; use async_trait::async_trait; use serde::{de::DeserializeOwned, Serialize}; @@ -278,9 +276,10 @@ pub trait Client { } /// `/genesis_chunked`: get genesis file in multiple chunks. + #[cfg(any(feature = "http-client", feature = "websocket-client"))] async fn genesis_chunked_stream( &self, - ) -> Pin, Error>> + '_>> { + ) -> core::pin::Pin, Error>> + '_>> { Box::pin(futures::stream::unfold(Some(0), move |chunk| async move { // Verify if there are more chunks to fetch let chunk = chunk?; From 945f17caffc71be7ff7a849134ebddf7c3c00b68 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Fri, 12 Jul 2024 14:22:18 +0200 Subject: [PATCH 9/9] Update changelog entry --- .changelog/unreleased/features/1438-genesis-chunked-endpoint.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/unreleased/features/1438-genesis-chunked-endpoint.md b/.changelog/unreleased/features/1438-genesis-chunked-endpoint.md index 9933439f2..e6502cdad 100644 --- a/.changelog/unreleased/features/1438-genesis-chunked-endpoint.md +++ b/.changelog/unreleased/features/1438-genesis-chunked-endpoint.md @@ -1,2 +1,2 @@ -- `[tendermint-rpc]` Add `/genesis_chunked` RPC endpoint +- `[tendermint-rpc]` Add support for the `/genesis_chunked` RPC endpoint ([\#1438](https://github.com/informalsystems/tendermint-rs/issues/1438)) \ No newline at end of file