Introduce Metadata type (#974)

* WIP new Metadata type

* Finish basic Metadata impl inc hashing and validation

* remove caching from metadata; can add that higher up

* remove caches

* update retain to use Metadata

* clippy fixes

* update codegen to use Metadata

* clippy

* WIP fixing subxt lib

* WIP fixing tests, rebuild artifacts, fix OrderedMap::retain

* get --all-targets compiling

* move DispatchError type lookup back to being optional

* cargo clippy

* fix docs

* re-use VariantIndex to get variants

* add docs and enforce docs on metadata crate

* fix docs

* add test and fix docs

* cargo fmt

* address review comments

* update lockfiles

* ExactSizeIter so we can ask for len() of things (and hopefully soon is_empty()

Cargo.toml

@@ -48,9 +48,9 @@ hex = "0.4.3"
 heck = "0.4.1"
 impl-serde = { version = "0.4.0" }
 jsonrpsee = { version = "0.16" }
-parking_lot = "0.12.0"
 pretty_assertions = "1.0.0"
 primitive-types = { version = "0.12.1", default-features = false, features = ["codec", "scale-info", "serde"] }
+proc-macro-error = "1.0.4"
 proc-macro2 = "1.0.58"
 quote = "1.0.27"
 regex = "1.8.1"
@@ -68,7 +68,6 @@ tracing = "0.1.34"
 tracing-wasm = "0.2.1"
 tracing-subscriber = "0.3.17"
 trybuild = "1.0.79"
-proc-macro-error = "1.0.4"
 wabt = "0.10.0"
 wasm-bindgen-test = "0.3.24"
 which = "4.4.0"

cli/src/commands/compatibility.rs

@@ -4,15 +4,12 @@
 
 use clap::Parser as ClapParser;
 use codec::Decode;
-use color_eyre::eyre::{self, WrapErr};
-use frame_metadata::{
-    v15::RuntimeMetadataV15, RuntimeMetadata, RuntimeMetadataPrefixed, META_RESERVED,
-};
+use color_eyre::eyre::WrapErr;
 use jsonrpsee::client_transport::ws::Uri;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use subxt_codegen::utils::MetadataVersion;
-use subxt_metadata::{get_pallet_hash, metadata_v14_to_latest, MetadataHasher};
+use subxt_metadata::Metadata;
 
 /// Verify metadata compatibility between substrate nodes.
 #[derive(Debug, ClapParser)]
@@ -66,9 +63,9 @@ async fn handle_pallet_metadata(
     for node in nodes.iter() {
         let metadata = fetch_runtime_metadata(node, version).await?;
 
-        match metadata.pallets.iter().find(|pallet| pallet.name == name) {
+        match metadata.pallet_by_name(name) {
             Some(pallet_metadata) => {
-                let hash = get_pallet_hash(&metadata.types, pallet_metadata);
+                let hash = pallet_metadata.hash();
                 let hex_hash = hex::encode(hash);
                 println!("Node {node:?} has pallet metadata hash {hex_hash:?}");
 
@@ -97,7 +94,7 @@ async fn handle_full_metadata(nodes: &[Uri], version: MetadataVersion) -> color_
     let mut compatibility_map: HashMap<String, Vec<String>> = HashMap::new();
     for node in nodes.iter() {
         let metadata = fetch_runtime_metadata(node, version).await?;
-        let hash = MetadataHasher::new().hash(&metadata);
+        let hash = metadata.hasher().hash();
         let hex_hash = hex::encode(hash);
         println!("Node {node:?} has metadata hash {hex_hash:?}",);
 
@@ -119,26 +116,8 @@ async fn handle_full_metadata(nodes: &[Uri], version: MetadataVersion) -> color_
 async fn fetch_runtime_metadata(
     url: &Uri,
     version: MetadataVersion,
-) -> color_eyre::Result<RuntimeMetadataV15> {
+) -> color_eyre::Result<Metadata> {
     let bytes = subxt_codegen::utils::fetch_metadata_bytes(url, version).await?;
-
-    let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
-    if metadata.0 != META_RESERVED {
-        return Err(eyre::eyre!(
-            "Node {:?} has invalid metadata prefix: {:?} expected prefix: {:?}",
-            url,
-            metadata.0,
-            META_RESERVED
-        ));
-    }
-
-    match metadata.1 {
-        RuntimeMetadata::V14(v14) => Ok(metadata_v14_to_latest(v14)),
-        RuntimeMetadata::V15(v15) => Ok(v15),
-        _ => Err(eyre::eyre!(
-            "Node {:?} with unsupported metadata version: {:?}",
-            url,
-            metadata.1
-        )),
-    }
+    let metadata = Metadata::decode(&mut &bytes[..])?;
+    Ok(metadata)
 }

cli/src/commands/explore/calls.rs

@@ -1,22 +1,23 @@
-use crate::utils::type_description::print_type_description;
-use crate::utils::type_example::print_type_examples;
-use crate::utils::with_indent;
 use clap::Args;
-
-use std::fmt::Write;
-use std::str::FromStr;
-use std::write;
-
 use color_eyre::eyre::eyre;
-use frame_metadata::v15::PalletMetadata;
-
 use scale_info::form::PortableForm;
 use scale_info::{PortableRegistry, Type, TypeDef, TypeDefVariant};
 use scale_value::{Composite, ValueDef};
+use std::fmt::Write;
+use std::str::FromStr;
+use std::write;
 
 use subxt::tx;
 use subxt::utils::H256;
-use subxt::{config::SubstrateConfig, Metadata, OfflineClient};
+use subxt::{
+    config::SubstrateConfig,
+    metadata::{types::PalletMetadata, Metadata},
+    OfflineClient,
+};
+
+use crate::utils::type_description::print_type_description;
+use crate::utils::type_example::print_type_examples;
+use crate::utils::with_indent;
 
 #[derive(Debug, Clone, Args)]
 pub struct CallsSubcommand {
@@ -28,13 +29,13 @@ pub struct CallsSubcommand {
 pub(crate) fn explore_calls(
     command: CallsSubcommand,
     metadata: &Metadata,
-    pallet_metadata: &PalletMetadata<PortableForm>,
+    pallet_metadata: PalletMetadata,
 ) -> color_eyre::Result<()> {
-    let pallet_name = pallet_metadata.name.as_str();
+    let pallet_name = pallet_metadata.name();
 
     // get the enum that stores the possible calls:
     let (calls_enum_type_def, _calls_enum_type) =
-        get_calls_enum_type(pallet_metadata, &metadata.runtime_metadata().types)?;
+        get_calls_enum_type(pallet_metadata, metadata.types())?;
 
     // if no call specified, show user the calls to choose from:
     let Some(call_name) = command.call else {
@@ -55,14 +56,9 @@ pub(crate) fn explore_calls(
 
     // if no trailing arguments specified show user the expected type of arguments with examples:
     if trailing_args.is_empty() {
-        let mut type_description =
-            print_type_description(&call.fields, &metadata.runtime_metadata().types)?;
+        let mut type_description = print_type_description(&call.fields, metadata.types())?;
         type_description = with_indent(type_description, 4);
-        let mut type_examples = print_type_examples(
-            &call.fields,
-            &metadata.runtime_metadata().types,
-            "SCALE_VALUE",
-        )?;
+        let mut type_examples = print_type_examples(&call.fields, metadata.types(), "SCALE_VALUE")?;
         type_examples = with_indent(type_examples, 4);
         let mut output = String::new();
         write!(output, "Usage:\n    subxt explore {pallet_name} calls {call_name} <SCALE_VALUE>\n        construct the call by providing a valid argument\n\n")?;
@@ -102,22 +98,19 @@ fn print_available_calls(pallet_calls: &TypeDefVariant<PortableForm>, pallet_nam
 }
 
 fn get_calls_enum_type<'a>(
-    pallet: &'a frame_metadata::v15::PalletMetadata<PortableForm>,
+    pallet: PalletMetadata,
     registry: &'a PortableRegistry,
 ) -> color_eyre::Result<(&'a TypeDefVariant<PortableForm>, &'a Type<PortableForm>)> {
-    let calls = pallet
-        .calls
-        .as_ref()
-        .ok_or(eyre!("The \"{}\" pallet has no calls.", pallet.name))?;
+    let call_ty = pallet
+        .call_ty_id()
+        .ok_or(eyre!("The \"{}\" pallet has no calls.", pallet.name()))?;
     let calls_enum_type = registry
-        .resolve(calls.ty.id)
-        .ok_or(eyre!("calls type with id {} not found.", calls.ty.id))?;
+        .resolve(call_ty)
+        .ok_or(eyre!("calls type with id {} not found.", call_ty))?;
+
     // should always be a variant type, where each variant corresponds to one call.
-    let calls_enum_type_def = match &calls_enum_type.type_def {
-        TypeDef::Variant(variant) => variant,
-        _ => {
-            return Err(eyre!("calls type is not a variant"));
-        }
+    let TypeDef::Variant(calls_enum_type_def) = &calls_enum_type.type_def else {
+        return Err(eyre!("calls type is not a variant"));
     };
     Ok((calls_enum_type_def, calls_enum_type))
 }

cli/src/commands/explore/constants.rs

@@ -1,16 +1,11 @@
-use crate::utils::type_description::print_type_description;
-use crate::utils::{print_docs_with_indent, with_indent};
 use clap::Args;
-
+use color_eyre::eyre::eyre;
 use std::fmt::Write;
 use std::write;
+use subxt::metadata::{types::PalletMetadata, Metadata};
 
-use color_eyre::eyre::eyre;
-use frame_metadata::v15::PalletMetadata;
-
-use scale_info::form::PortableForm;
-
-use subxt::Metadata;
+use crate::utils::type_description::print_type_description;
+use crate::utils::{print_docs_with_indent, with_indent};
 
 #[derive(Debug, Clone, Args)]
 pub struct ConstantsSubcommand {
@@ -20,17 +15,17 @@ pub struct ConstantsSubcommand {
 pub(crate) fn explore_constants(
     command: ConstantsSubcommand,
     metadata: &Metadata,
-    pallet_metadata: &PalletMetadata<PortableForm>,
+    pallet_metadata: PalletMetadata,
 ) -> color_eyre::Result<()> {
-    let pallet_name = pallet_metadata.name.as_str();
+    let pallet_name = pallet_metadata.name();
     let Some(constant_name) = command.constant else {
         let available_constants = print_available_constants(pallet_metadata, pallet_name);
         println!("Usage:\n    subxt explore {pallet_name} constants <CONSTANT>\n        explore a specific call within this pallet\n\n{available_constants}", );
         return Ok(());
     };
 
     // if specified constant is wrong, show user the constants to choose from (but this time as an error):
-    let Some(constant) = pallet_metadata.constants.iter().find(|constant| constant.name.to_lowercase() == constant_name.to_lowercase())   else {
+    let Some(constant) = pallet_metadata.constants().find(|constant| constant.name().to_lowercase() == constant_name.to_lowercase()) else {
         let available_constants = print_available_constants(pallet_metadata, pallet_name);
         let description = format!("Usage:\n    subxt explore {pallet_name} constants <CONSTANT>\n        explore a specific call within this pallet\n\n{available_constants}", );
         let err = eyre!("constant \"{constant_name}\" not found in \"{pallet_name}\" pallet!\n\n{description}");
@@ -39,25 +34,22 @@ pub(crate) fn explore_constants(
 
     // docs
     let mut output = String::new();
-    let doc_string = print_docs_with_indent(&constant.docs, 4);
+    let doc_string = print_docs_with_indent(constant.docs(), 4);
     if !doc_string.is_empty() {
         write!(output, "Description:\n{doc_string}")?;
     }
 
     // shape
-    let mut type_description = print_type_description(&constant.ty.id, metadata.types())?;
+    let mut type_description = print_type_description(&constant.ty(), metadata.types())?;
     type_description = with_indent(type_description, 4);
     write!(
         output,
         "\n\nThe constant has the following shape:\n{type_description}"
     )?;
 
     // value
-    let scale_val = scale_value::scale::decode_as_type(
-        &mut &constant.value[..],
-        constant.ty.id,
-        metadata.types(),
-    )?;
+    let scale_val =
+        scale_value::scale::decode_as_type(&mut constant.value(), constant.ty(), metadata.types())?;
     write!(
         output,
         "\n\nThe value of the constant is:\n    {}",
@@ -68,15 +60,12 @@ pub(crate) fn explore_constants(
     Ok(())
 }
 
-fn print_available_constants(
-    pallet_metadata: &PalletMetadata<PortableForm>,
-    pallet_name: &str,
-) -> String {
-    if pallet_metadata.constants.is_empty() {
+fn print_available_constants(pallet_metadata: PalletMetadata, pallet_name: &str) -> String {
+    if pallet_metadata.constants().len() == 0 {
         return format!("No <CONSTANT>'s available in the \"{pallet_name}\" pallet.");
     }
     let mut output = format!("Available <CONSTANT>'s in the \"{pallet_name}\" pallet:");
-    let mut strings: Vec<_> = pallet_metadata.constants.iter().map(|c| &c.name).collect();
+    let mut strings: Vec<_> = pallet_metadata.constants().map(|c| c.name()).collect();
     strings.sort();
     for constant in strings {
         output.push_str("\n    ");

cli/src/commands/explore/mod.rs

@@ -7,10 +7,6 @@ use std::write;
 
 use codec::Decode;
 use color_eyre::eyre::eyre;
-use frame_metadata::v15::RuntimeMetadataV15;
-use frame_metadata::RuntimeMetadataPrefixed;
-
-use syn::__private::str;
 
 use crate::commands::explore::calls::{explore_calls, CallsSubcommand};
 use crate::commands::explore::constants::{explore_constants, ConstantsSubcommand};
@@ -89,24 +85,23 @@ pub enum PalletSubcommand {
 pub async fn run(opts: Opts) -> color_eyre::Result<()> {
     // get the metadata
     let bytes = opts.file_or_url.fetch().await?;
-    let metadata_prefixed = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
-    let metadata = Metadata::try_from(metadata_prefixed)?;
+    let metadata = Metadata::decode(&mut &bytes[..])?;
 
     // if no pallet specified, show user the pallets to choose from:
     let Some(pallet_name) = opts.pallet else {
-        let available_pallets = print_available_pallets(metadata.runtime_metadata());
+        let available_pallets = print_available_pallets(&metadata);
         println!("Usage:\n    subxt explore <PALLET>\n        explore a specific pallet\n\n{available_pallets}", );
         return Ok(());
     };
 
     // if specified pallet is wrong, show user the pallets to choose from (but this time as an error):
-    let Some(pallet_metadata) = metadata.runtime_metadata().pallets.iter().find(|pallet| pallet.name.to_lowercase() == pallet_name.to_lowercase())else {
-        return Err(eyre!("pallet \"{}\" not found in metadata!\n{}", pallet_name, print_available_pallets(metadata.runtime_metadata())));
+    let Some(pallet_metadata) = metadata.pallets().find(|pallet| pallet.name().to_lowercase() == pallet_name.to_lowercase()) else {
+        return Err(eyre!("pallet \"{}\" not found in metadata!\n{}", pallet_name, print_available_pallets(&metadata)));
     };
 
     // if correct pallet was specified but no subcommand, instruct the user how to proceed:
     let Some(pallet_subcomand) = opts.pallet_subcommand else {
-        let docs_string = print_docs_with_indent(&pallet_metadata.docs, 4);
+        let docs_string = print_docs_with_indent(pallet_metadata.docs(), 4);
         let mut output = String::new();
         if !docs_string.is_empty() {
             write!(output, "Description:\n{docs_string}")?;
@@ -132,12 +127,12 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
     }
 }
 
-fn print_available_pallets(metadata_v15: &RuntimeMetadataV15) -> String {
-    if metadata_v15.pallets.is_empty() {
+fn print_available_pallets(metadata: &Metadata) -> String {
+    if metadata.pallets().len() == 0 {
         "There are no <PALLET> values available.".to_string()
     } else {
         let mut output = "Available <PALLET> values are:".to_string();
-        let mut strings: Vec<_> = metadata_v15.pallets.iter().map(|p| &p.name).collect();
+        let mut strings: Vec<_> = metadata.pallets().map(|p| p.name()).collect();
         strings.sort();
         for pallet in strings {
             write!(output, "\n    {}", pallet).unwrap();