Add runtime_metadata_url to pull metadata directly from a node

cli/src/commands/codegen.rs

@@ -4,12 +4,7 @@
 
 use clap::Parser as ClapParser;
 use color_eyre::eyre;
-use frame_metadata::RuntimeMetadataPrefixed;
 use jsonrpsee::client_transport::ws::Uri;
-use scale::{
-    Decode,
-    Input,
-};
 use std::{
     fs,
     io::Read,
@@ -48,7 +43,7 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
         let mut file = fs::File::open(file)?;
         let mut bytes = Vec::new();
         file.read_to_end(&mut bytes)?;
-        codegen(&mut &bytes[..], opts.derives, opts.crate_path)?;
+        codegen(&bytes, opts.derives, opts.crate_path)?;
         return Ok(())
     }
 
@@ -57,18 +52,16 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
             .parse::<Uri>()
             .expect("default url is valid")
     });
-    let (_, bytes) = super::metadata::fetch_metadata(&url).await?;
-    codegen(&mut &bytes[..], opts.derives, opts.crate_path)?;
+    let bytes = subxt_codegen::utils::fetch_metadata_bytes(&url).await?;
+    codegen(&bytes, opts.derives, opts.crate_path)?;
     Ok(())
 }
 
-fn codegen<I: Input>(
-    encoded: &mut I,
+fn codegen(
+    metadata_bytes: &[u8],
     raw_derives: Vec<String>,
     crate_path: Option<String>,
 ) -> color_eyre::Result<()> {
-    let metadata = <RuntimeMetadataPrefixed as Decode>::decode(encoded)?;
-    let generator = subxt_codegen::RuntimeGenerator::new(metadata);
     let item_mod = syn::parse_quote!(
         pub mod api {}
     );
@@ -82,7 +75,12 @@ fn codegen<I: Input>(
     let mut derives = DerivesRegistry::new(&crate_path);
     derives.extend_for_all(p.into_iter());
 
-    let runtime_api = generator.generate_runtime(item_mod, derives, crate_path);
+    let runtime_api = subxt_codegen::generate_runtime_api_from_bytes(
+        item_mod,
+        metadata_bytes,
+        derives,
+        crate_path,
+    );
     println!("{}", runtime_api);
     Ok(())
 }

cli/src/commands/compatibility.rs

@@ -111,7 +111,7 @@ async fn handle_full_metadata(nodes: &[Uri]) -> color_eyre::Result<()> {
 }
 
 async fn fetch_runtime_metadata(url: &Uri) -> color_eyre::Result<RuntimeMetadataV14> {
-    let (_, bytes) = super::metadata::fetch_metadata(url).await?;
+    let bytes = subxt_codegen::utils::fetch_metadata_bytes(url).await?;
 
     let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
     if metadata.0 != META_RESERVED {

cli/src/commands/metadata.rs

@@ -5,24 +5,13 @@
 use clap::Parser as ClapParser;
 use color_eyre::eyre;
 use frame_metadata::RuntimeMetadataPrefixed;
-use jsonrpsee::{
-    async_client::ClientBuilder,
-    client_transport::ws::{
-        Uri,
-        WsTransportClientBuilder,
-    },
-    core::{
-        client::ClientT,
-        Error,
-    },
-    http_client::HttpClientBuilder,
-    rpc_params,
-};
+use jsonrpsee::client_transport::ws::Uri;
 use scale::Decode;
 use std::io::{
     self,
     Write,
 };
+use subxt_codegen::utils::fetch_metadata_hex;
 
 /// Download metadata from a substrate node, for use with `subxt` codegen.
 #[derive(Debug, ClapParser)]
@@ -41,10 +30,11 @@ pub struct Opts {
 }
 
 pub async fn run(opts: Opts) -> color_eyre::Result<()> {
-    let (hex_data, bytes) = fetch_metadata(&opts.url).await?;
+    let hex_data = fetch_metadata_hex(&opts.url).await?;
 
     match opts.format.as_str() {
         "json" => {
+            let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
             let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
             let json = serde_json::to_string_pretty(&metadata)?;
             println!("{}", json);
@@ -54,7 +44,10 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
             println!("{}", hex_data);
             Ok(())
         }
-        "bytes" => Ok(io::stdout().write_all(&bytes)?),
+        "bytes" => {
+            let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
+            Ok(io::stdout().write_all(&bytes)?)
+        }
         _ => {
             Err(eyre::eyre!(
                 "Unsupported format `{}`, expected `json`, `hex` or `bytes`",
@@ -63,40 +56,3 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
         }
     }
 }
-
-pub async fn fetch_metadata(url: &Uri) -> color_eyre::Result<(String, Vec<u8>)> {
-    let hex_data = match url.scheme_str() {
-        Some("http") => fetch_metadata_http(url).await,
-        Some("ws") | Some("wss") => fetch_metadata_ws(url).await,
-        invalid_scheme => {
-            let scheme = invalid_scheme.unwrap_or("no scheme");
-            Err(eyre::eyre!(format!(
-                "`{}` not supported, expects 'http', 'ws', or 'wss'",
-                scheme
-            )))
-        }
-    }?;
-
-    let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
-
-    Ok((hex_data, bytes))
-}
-
-async fn fetch_metadata_ws(url: &Uri) -> color_eyre::Result<String> {
-    let (sender, receiver) = WsTransportClientBuilder::default()
-        .build(url.to_string().parse::<Uri>().unwrap())
-        .await
-        .map_err(|e| Error::Transport(e.into()))?;
-
-    let client = ClientBuilder::default()
-        .max_notifs_per_subscription(4096)
-        .build_with_tokio(sender, receiver);
-
-    Ok(client.request("state_getMetadata", rpc_params![]).await?)
-}
-
-async fn fetch_metadata_http(url: &Uri) -> color_eyre::Result<String> {
-    let client = HttpClientBuilder::default().build(url.to_string())?;
-
-    Ok(client.request::<String>("state_getMetadata", None).await?)
-}

codegen/Cargo.toml

@@ -21,6 +21,9 @@ quote = "1.0.8"
 syn = "1.0.58"
 scale-info = { version = "2.0.0", features = ["bit-vec"] }
 subxt-metadata = { version = "0.24.0", path = "../metadata" }
+jsonrpsee = { version = "0.15.1", features = ["async-client", "client-ws-transport", "http-client"] }
+hex = "0.4.3"
+tokio = { version = "1.8", features = ["macros", "rt-multi-thread"] }
 
 [dev-dependencies]
 bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }

codegen/src/api/mod.rs

@@ -19,6 +19,10 @@ use crate::{
         CompositeDefFields,
         TypeGenerator,
     },
+    utils::{
+        fetch_metadata_bytes_blocking,
+        Uri,
+    },
     CratePath,
 };
 use codec::Decode;
@@ -50,9 +54,10 @@ use syn::parse_quote;
 /// * `item_mod` - The module declaration for which the API is implemented.
 /// * `path` - The path to the scale encoded metadata of the runtime node.
 /// * `derives` - Provide custom derives for the generated types.
+/// * `crate_path` - Path to the `subxt` crate.
 ///
 /// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases.
-pub fn generate_runtime_api<P>(
+pub fn generate_runtime_api_from_path<P>(
     item_mod: syn::ItemMod,
     path: P,
     derives: DerivesRegistry,
@@ -69,6 +74,48 @@ where
     file.read_to_end(&mut bytes)
         .unwrap_or_else(|e| abort_call_site!("Failed to read metadata file: {}", e));
 
+    generate_runtime_api_from_bytes(item_mod, &bytes, derives, crate_path)
+}
+
+/// Generates the API for interacting with a substrate runtime, using metadata
+/// that can be downloaded from a node at the provided URL.
+///
+/// # Arguments
+///
+/// * `item_mod` - The module declaration for which the API is implemented.
+/// * `url` - HTTP/WS URL to the substrate node you'd like to pull metadata from.
+/// * `derives` - Provide custom derives for the generated types.
+/// * `crate_path` - Path to the `subxt` crate.
+///
+/// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases.
+pub fn generate_runtime_api_from_url(
+    item_mod: syn::ItemMod,
+    url: &Uri,
+    derives: DerivesRegistry,
+    crate_path: CratePath,
+) -> TokenStream2 {
+    let bytes = fetch_metadata_bytes_blocking(url)
+        .unwrap_or_else(|e| abort_call_site!("Failed to obtain metadata: {}", e));
+
+    generate_runtime_api_from_bytes(item_mod, &bytes, derives, crate_path)
+}
+
+/// Generates the API for interacting with a substrate runtime, using metadata bytes
+///
+/// # Arguments
+///
+/// * `item_mod` - The module declaration for which the API is implemented.
+/// * `url` - HTTP/WS URL to the substrate node you'd like to pull metadata from.
+/// * `derives` - Provide custom derives for the generated types.
+/// * `crate_path` - Path to the `subxt` crate.
+///
+/// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases.
+pub fn generate_runtime_api_from_bytes(
+    item_mod: syn::ItemMod,
+    bytes: &[u8],
+    derives: DerivesRegistry,
+    crate_path: CratePath,
+) -> TokenStream2 {
     let metadata = frame_metadata::RuntimeMetadataPrefixed::decode(&mut &bytes[..])
         .unwrap_or_else(|e| abort_call_site!("Failed to decode metadata: {}", e));
 

codegen/src/lib.rs

@@ -45,9 +45,13 @@ mod api;
 mod ir;
 mod types;
 
+pub mod utils;
+
 pub use self::{
     api::{
-        generate_runtime_api,
+        generate_runtime_api_from_bytes,
+        generate_runtime_api_from_path,
+        generate_runtime_api_from_url,
         RuntimeGenerator,
     },
     types::{

codegen/src/utils/fetch_metadata.rs

@@ -0,0 +1,107 @@
+use jsonrpsee::{
+    async_client::ClientBuilder,
+    client_transport::ws::{
+        Uri,
+        WsTransportClientBuilder,
+    },
+    core::{
+        client::ClientT,
+        Error,
+    },
+    http_client::HttpClientBuilder,
+    rpc_params,
+};
+
+/// Returns the metadata bytes from the provided URL, blocking the current thread.
+pub fn fetch_metadata_bytes_blocking(url: &Uri) -> Result<Vec<u8>, FetchMetadataError> {
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+        .block_on(fetch_metadata_bytes(url))
+}
+
+/// Returns the raw, 0x prefixed metadata hex from the provided URL, blocking the current thread.
+pub fn fetch_metadata_hex_blocking(url: &Uri) -> Result<String, FetchMetadataError> {
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+        .block_on(fetch_metadata_hex(url))
+}
+
+/// Returns the metadata bytes from the provided URL.
+pub async fn fetch_metadata_bytes(url: &Uri) -> Result<Vec<u8>, FetchMetadataError> {
+    let hex = fetch_metadata_hex(url).await?;
+    let bytes = hex::decode(hex.trim_start_matches("0x"))?;
+    Ok(bytes)
+}
+
+/// Returns the raw, 0x prefixed metadata hex from the provided URL.
+pub async fn fetch_metadata_hex(url: &Uri) -> Result<String, FetchMetadataError> {
+    let hex_data = match url.scheme_str() {
+        Some("http") => fetch_metadata_http(url).await,
+        Some("ws") | Some("wss") => fetch_metadata_ws(url).await,
+        invalid_scheme => {
+            let scheme = invalid_scheme.unwrap_or("no scheme");
+            Err(FetchMetadataError::InvalidScheme(scheme.to_owned()))
+        }
+    }?;
+    Ok(hex_data)
+}
+
+async fn fetch_metadata_ws(url: &Uri) -> Result<String, FetchMetadataError> {
+    let (sender, receiver) = WsTransportClientBuilder::default()
+        .build(url.to_string().parse::<Uri>().unwrap())
+        .await
+        .map_err(|e| Error::Transport(e.into()))?;
+
+    let client = ClientBuilder::default()
+        .max_notifs_per_subscription(4096)
+        .build_with_tokio(sender, receiver);
+
+    Ok(client.request("state_getMetadata", rpc_params![]).await?)
+}
+
+async fn fetch_metadata_http(url: &Uri) -> Result<String, FetchMetadataError> {
+    let client = HttpClientBuilder::default().build(url.to_string())?;
+
+    Ok(client.request::<String>("state_getMetadata", None).await?)
+}
+
+#[derive(Debug)]
+pub enum FetchMetadataError {
+    DecodeError(hex::FromHexError),
+    RequestError(jsonrpsee::core::Error),
+    InvalidScheme(String),
+}
+
+impl std::fmt::Display for FetchMetadataError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FetchMetadataError::DecodeError(e) => {
+                write!(f, "Cannot decode hex value: {e}")
+            }
+            FetchMetadataError::RequestError(e) => write!(f, "Request error: {e}"),
+            FetchMetadataError::InvalidScheme(s) => {
+                write!(
+                    f,
+                    "'{s}' not supported, supported URl schemes are http, ws or wss."
+                )
+            }
+        }
+    }
+}
+
+impl std::error::Error for FetchMetadataError {}
+
+impl From<hex::FromHexError> for FetchMetadataError {
+    fn from(e: hex::FromHexError) -> Self {
+        FetchMetadataError::DecodeError(e)
+    }
+}
+impl From<jsonrpsee::core::Error> for FetchMetadataError {
+    fn from(e: jsonrpsee::core::Error) -> Self {
+        FetchMetadataError::RequestError(e)
+    }
+}

codegen/src/utils/mod.rs

@@ -0,0 +1,12 @@
+mod fetch_metadata;
+
+// easy access to this type needed for fetching metadata:
+pub use jsonrpsee::client_transport::ws::Uri;
+
+pub use fetch_metadata::{
+    fetch_metadata_bytes,
+    fetch_metadata_bytes_blocking,
+    fetch_metadata_hex,
+    fetch_metadata_hex_blocking,
+    FetchMetadataError,
+};

examples/examples/custom_metadata_url.rs

@@ -0,0 +1,11 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+// If you'd like to use metadata directly from a running node, you
+// can provide a URL to that node here. Note that if the metadata cannot
+// be retrieved from this node URL at compile time, compilation will fail.
+#[subxt::subxt(runtime_metadata_url = "wss://rpc.polkadot.io:443")]
+pub mod polkadot {}
+
+fn main() {}