cli: Command to fetch chainSpec and optimise its size (#1278)

* cli: Add chainSpec command Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli/chainSpec: Move to dedicated module Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Compute the state root hash Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Remove code substitutes Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * artifacts: Update polkadot.json Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * scripts: Generate the chain spec Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Remove testing artifacts Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Fix clippy Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Apply rustfmt Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Introduce feature flag for smoldot dependency Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Rename chain-spec to chain-spec-pruning Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * scripts: Update chain-spec command Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> --------- Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>
paritytech · Nov 29, 2023 · 0724735 · 0724735
1 parent 44c42c3
commit 0724735
Show file tree

Hide file tree

Showing 8 changed files with 270 additions and 46 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/artifacts/demo_chain_specs/polkadot.json b/artifacts/demo_chain_specs/polkadot.json
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
@@ -16,6 +16,11 @@ description = "Command line utilities for working with subxt codegen"
 name = "subxt"
 path = "src/main.rs"
 
+[features]
+# Compute the state root hash from the genesis entry.
+# Enable this to create a smaller chain spec file.
+chain-spec-pruning = ["smoldot"]
+
 [dependencies]
 subxt-codegen = { workspace = true, features = ["fetch-metadata"] }
 subxt-metadata = { workspace = true }
@@ -32,3 +37,5 @@ scale-value = { workspace = true }
 syn = { workspace = true }
 jsonrpsee = { workspace = true, features = ["async-client", "client-ws-transport-native-tls", "http-client"] }
 tokio = { workspace = true, features = ["rt-multi-thread"] }
+thiserror = { workspace = true }
+smoldot = { workspace = true, optional = true}
diff --git a/cli/src/commands/chain_spec/fetch.rs b/cli/src/commands/chain_spec/fetch.rs
@@ -0,0 +1,63 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use jsonrpsee::{
+    async_client::ClientBuilder,
+    client_transport::ws::WsTransportClientBuilder,
+    core::{client::ClientT, Error},
+    http_client::HttpClientBuilder,
+};
+use std::time::Duration;
+use subxt_codegen::fetch_metadata::Url;
+
+/// Returns the node's chainSpec from the provided URL.
+pub async fn fetch_chain_spec(url: Url) -> Result<serde_json::Value, FetchSpecError> {
+    async fn fetch_ws(url: Url) -> Result<serde_json::Value, Error> {
+        let (sender, receiver) = WsTransportClientBuilder::default()
+            .build(url)
+            .await
+            .map_err(|e| Error::Transport(e.into()))?;
+
+        let client = ClientBuilder::default()
+            .request_timeout(Duration::from_secs(180))
+            .max_buffer_capacity_per_subscription(4096)
+            .build_with_tokio(sender, receiver);
+
+        inner_fetch(client).await
+    }
+
+    async fn fetch_http(url: Url) -> Result<serde_json::Value, Error> {
+        let client = HttpClientBuilder::default()
+            .request_timeout(Duration::from_secs(180))
+            .build(url)?;
+
+        inner_fetch(client).await
+    }
+
+    async fn inner_fetch(client: impl ClientT) -> Result<serde_json::Value, Error> {
+        client
+            .request("sync_state_genSyncSpec", jsonrpsee::rpc_params![true])
+            .await
+    }
+
+    let spec = match url.scheme() {
+        "http" | "https" => fetch_http(url).await.map_err(FetchSpecError::RequestError),
+        "ws" | "wss" => fetch_ws(url).await.map_err(FetchSpecError::RequestError),
+        invalid_scheme => Err(FetchSpecError::InvalidScheme(invalid_scheme.to_owned())),
+    }?;
+
+    Ok(spec)
+}
+
+/// Error attempting to fetch chainSpec.
+#[derive(Debug, thiserror::Error)]
+#[non_exhaustive]
+pub enum FetchSpecError {
+    /// JSON-RPC error fetching metadata.
+    #[error("Request error: {0}")]
+    RequestError(#[from] jsonrpsee::core::Error),
+    /// URL scheme is not http, https, ws or wss.
+    #[error("'{0}' not supported, supported URI schemes are http, https, ws or wss.")]
+    InvalidScheme(String),
+}
diff --git a/cli/src/commands/chain_spec/mod.rs b/cli/src/commands/chain_spec/mod.rs
@@ -0,0 +1,131 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use clap::Parser as ClapParser;
+#[cfg(feature = "chain-spec-pruning")]
+use serde_json::Value;
+use std::{io::Write, path::PathBuf};
+use subxt_codegen::fetch_metadata::Url;
+
+mod fetch;
+
+/// Download chainSpec from a substrate node.
+#[derive(Debug, ClapParser)]
+pub struct Opts {
+    /// The url of the substrate node to query for metadata for codegen.
+    #[clap(long)]
+    url: Url,
+    /// Write the output of the command to the provided file path.
+    #[clap(long, short, value_parser)]
+    output_file: Option<PathBuf>,
+    /// Replaced the genesis raw entry with a stateRootHash to optimize
+    /// the spec size and avoid the need to calculate the genesis storage.
+    ///
+    /// This option is enabled with the `chain-spec-pruning` feature.
+    ///
+    /// Defaults to `false`.
+    #[cfg(feature = "chain-spec-pruning")]
+    #[clap(long)]
+    state_root_hash: bool,
+    /// Remove the `codeSubstitutes` entry from the chain spec.
+    /// This is useful when wanting to store a smaller chain spec.
+    /// At this moment, the light client does not utilize this object.
+    ///
+    /// Defaults to `false`.
+    #[clap(long)]
+    remove_substitutes: bool,
+}
+
+/// Error attempting to fetch chainSpec.
+#[derive(Debug, thiserror::Error)]
+#[non_exhaustive]
+pub enum ChainSpecError {
+    /// Failed to fetch the chain spec.
+    #[error("Failed to fetch the chain spec: {0}")]
+    FetchError(#[from] fetch::FetchSpecError),
+
+    #[cfg(feature = "chain-spec-pruning")]
+    /// The provided chain spec is invalid.
+    #[error("Error while parsing the chain spec: {0})")]
+    ParseError(String),
+
+    #[cfg(feature = "chain-spec-pruning")]
+    /// Cannot compute the state root hash.
+    #[error("Error computing state root hash: {0})")]
+    ComputeError(String),
+
+    /// Other error.
+    #[error("Other: {0})")]
+    Other(String),
+}
+
+#[cfg(feature = "chain-spec-pruning")]
+fn compute_state_root_hash(spec: &Value) -> Result<[u8; 32], ChainSpecError> {
+    let chain_spec = smoldot::chain_spec::ChainSpec::from_json_bytes(spec.to_string().as_bytes())
+        .map_err(|err| ChainSpecError::ParseError(err.to_string()))?;
+
+    let genesis_chain_information = chain_spec.to_chain_information().map(|(ci, _)| ci);
+
+    let state_root = match genesis_chain_information {
+        Ok(genesis_chain_information) => {
+            let header = genesis_chain_information.as_ref().finalized_block_header;
+            *header.state_root
+        }
+        // From the smoldot code this error is encountered when the genesis already contains the
+        // state root hash entry instead of the raw entry.
+        Err(smoldot::chain_spec::FromGenesisStorageError::UnknownStorageItems) => *chain_spec
+            .genesis_storage()
+            .into_trie_root_hash()
+            .ok_or_else(|| {
+                ChainSpecError::ParseError(
+                    "The chain spec does not contain the proper shape for the genesis.raw entry"
+                        .to_string(),
+                )
+            })?,
+        Err(err) => return Err(ChainSpecError::ComputeError(err.to_string())),
+    };
+
+    Ok(state_root)
+}
+
+pub async fn run(opts: Opts, output: &mut impl Write) -> color_eyre::Result<()> {
+    let url = opts.url;
+
+    let mut spec = fetch::fetch_chain_spec(url).await?;
+
+    let mut output: Box<dyn Write> = match opts.output_file {
+        Some(path) => Box::new(std::fs::File::create(path)?),
+        None => Box::new(output),
+    };
+
+    #[cfg(feature = "chain-spec-pruning")]
+    if opts.state_root_hash {
+        let state_root_hash = compute_state_root_hash(&spec)?;
+        let state_root_hash = format!("0x{}", hex::encode(state_root_hash));
+
+        if let Some(genesis) = spec.get_mut("genesis") {
+            let object = genesis.as_object_mut().ok_or_else(|| {
+                ChainSpecError::Other("The genesis entry must be an object".to_string())
+            })?;
+
+            object.remove("raw").ok_or_else(|| {
+                ChainSpecError::Other("The genesis entry must contain a raw entry".to_string())
+            })?;
+
+            object.insert("stateRootHash".to_string(), Value::String(state_root_hash));
+        }
+    }
+
+    if opts.remove_substitutes {
+        let object = spec
+            .as_object_mut()
+            .ok_or_else(|| ChainSpecError::Other("The chain spec must be an object".to_string()))?;
+
+        object.remove("codeSubstitutes");
+    }
+
+    let json = serde_json::to_string_pretty(&spec)?;
+    write!(output, "{json}")?;
+    Ok(())
+}
diff --git a/cli/src/commands/mod.rs b/cli/src/commands/mod.rs
@@ -2,6 +2,7 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
+pub mod chain_spec;
 pub mod codegen;
 pub mod compatibility;
 pub mod diff;

diff --git a/cli/src/main.rs b/cli/src/main.rs
@@ -18,6 +18,7 @@ enum Command {
     Diff(commands::diff::Opts),
     Version(commands::version::Opts),
     Explore(commands::explore::Opts),
+    ChainSpec(commands::chain_spec::Opts),
 }
 
 #[tokio::main]
@@ -32,5 +33,6 @@ async fn main() -> color_eyre::Result<()> {
         Command::Diff(opts) => commands::diff::run(opts, &mut output).await,
         Command::Version(opts) => commands::version::run(opts, &mut output),
         Command::Explore(opts) => commands::explore::run(opts, &mut output).await,
+        Command::ChainSpec(opts) => commands::chain_spec::run(opts, &mut output).await,
     }
 }
diff --git a/scripts/artifacts.sh b/scripts/artifacts.sh
@@ -21,3 +21,6 @@ cargo run --bin subxt metadata --file artifacts/polkadot_metadata_full.scale --p
 cargo run --bin subxt metadata --file artifacts/polkadot_metadata_full.scale --pallets "" > artifacts/polkadot_metadata_tiny.scale
 # generate a metadata file that only contains some custom metadata
 cargo run --bin generate-custom-metadata > artifacts/metadata_with_custom_values.scale 
+
+# Generate the polkadot chain spec.
+cargo run --features chain-spec-pruning --bin subxt chain-spec --url wss://rpc.polkadot.io:443 --output-file artifacts/demo_chain_specs/polkadot.json --state-root-hash --remove-substitutes