runtime-interface: Implement register_only functions (#10640)

* runtime-interface: Implement `register_only` functions

The runtime interface supports versioning of functions. Currently, if you add a new function it will
be used by the runtime automatically. This results in requiring all nodes of a network to upgrade
before the runtime is upgraded, otherwise they will fail to instantiate the new runtime because of
missing host functions. This pr introduces `register_only` functions. This can be used when a new
runtime interface function should be introduced, but the actual usage can be deferred. This means
that nodes will have the host function for this, but the runtime will still use the old version of
the function when being compiled for wasm. However, when a runtime is enacted that uses the new host
function, the "old nodes" will already have the host function and will continue to work.

* Update primitives/runtime-interface/src/lib.rs

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

* Update primitives/runtime-interface/proc-macro/src/utils.rs

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

* FMT

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs

@@ -52,7 +52,7 @@ pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool, tracing: bool) -> Res
  let runtime_interface = get_runtime_interface(trait_def)?;
 
  // latest version dispatch
- let token_stream: Result<TokenStream> = runtime_interface.latest_versions().try_fold(
+ let token_stream: Result<TokenStream> = runtime_interface.latest_versions_to_call().try_fold(
  TokenStream::new(),
  |mut t, (latest_version, method)| {
  t.extend(function_for_method(method, latest_version, is_wasm_only)?);

primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs

@@ -45,19 +45,18 @@ use std::iter::Iterator;
 /// implementations for the host functions on the host.
 pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
  let trait_name = &trait_def.ident;
- let extern_host_function_impls = get_runtime_interface(trait_def)?.latest_versions().try_fold(
- TokenStream::new(),
- |mut t, (version, method)| {
+ let extern_host_function_impls = get_runtime_interface(trait_def)?
+ .latest_versions_to_call()
+ .try_fold(TokenStream::new(), |mut t, (version, method)| {
  t.extend(generate_extern_host_function(method, version, trait_name)?);
  Ok::<_, Error>(t)
- },
- )?;
+ })?;
  let exchangeable_host_functions = get_runtime_interface(trait_def)?
- .latest_versions()
+ .latest_versions_to_call()
  .try_fold(TokenStream::new(), |mut t, (_, m)| {
-  t.extend(generate_exchangeable_host_function(m)?);
-  Ok::<_, Error>(t)
-  })?;
+ t.extend(generate_exchangeable_host_function(m)?);
+ Ok::<_, Error>(t)
+ })?;
  let host_functions_struct = generate_host_functions_struct(trait_def, is_wasm_only)?;
 
  Ok(quote! {

primitives/runtime-interface/proc-macro/src/utils.rs

@@ -20,8 +20,8 @@
 use proc_macro2::{Span, TokenStream};
 
 use syn::{
- parse_quote, spanned::Spanned, token, Attribute, Error, FnArg, Ident, ItemTrait, Lit, Meta,
- NestedMeta, Pat, PatType, Result, Signature, TraitItem, TraitItemMethod, Type,
+ parse::Parse, parse_quote, spanned::Spanned, token, Error, FnArg, Ident, ItemTrait, LitInt,
+ Pat, PatType, Result, Signature, TraitItem, TraitItemMethod, Type,
 };
 
 use proc_macro_crate::{crate_name, FoundCrate};
@@ -35,31 +35,65 @@ use quote::quote;
 
 use inflector::Inflector;
 
+mod attributes {
+ syn::custom_keyword!(register_only);
+}
+
 /// Runtime interface function with all associated versions of this function.
 pub struct RuntimeInterfaceFunction<'a> {
- latest_version: u32,
+ latest_version_to_call: Option<u32>,
  versions: BTreeMap<u32, &'a TraitItemMethod>,
 }
 
 impl<'a> RuntimeInterfaceFunction<'a> {
- fn new(version: u32, trait_item: &'a TraitItemMethod) -> Self {
+ fn new(version: VersionAttribute, trait_item: &'a TraitItemMethod) -> Self {
  Self {
- latest_version: version,
- versions: {
- let mut res = BTreeMap::new();
- res.insert(version, trait_item);
- res
- },
+ latest_version_to_call: version.is_callable().then(|| version.version),
+ versions: BTreeMap::from([(version.version, trait_item)]),
  }
  }
 
- pub fn latest_version(&self) -> (u32, &TraitItemMethod) {
- (
- self.latest_version,
- self.versions.get(&self.latest_version).expect(
- "If latest_version has a value, the key with this value is in the versions; qed",
+ /// Returns the latest version of this runtime interface function plus the actual function
+ /// implementation.
+ ///
+ /// This isn't required to be the latest version, because a runtime interface function can be
+ /// annotated with `register_only` to ensure that the host exposes the host function but it
+ /// isn't used when compiling the runtime.
+ pub fn latest_version_to_call(&self) -> Option<(u32, &TraitItemMethod)> {
+ self.latest_version_to_call.map(|v| {
+ (
+ v,
+ *self.versions.get(&v).expect(
+ "If latest_version_to_call has a value, the key with this value is in the versions; qed",
  ),
  )
+ })
+ }
+
+ /// Add a different version of the function.
+ fn add_version(
+ &mut self,
+ version: VersionAttribute,
+ trait_item: &'a TraitItemMethod,
+ ) -> Result<()> {
+ if let Some(existing_item) = self.versions.get(&version.version) {
+ let mut err = Error::new(trait_item.span(), "Duplicated version attribute");
+ err.combine(Error::new(
+ existing_item.span(),
+ "Previous version with the same number defined here",
+ ));
+
+ return Err(err)
+ }
+
+ self.versions.insert(version.version, trait_item);
+ if self.latest_version_to_call.map_or(true, |v| v < version.version) &&
+ version.is_callable()
+ {
+ self.latest_version_to_call = Some(version.version);
+ }
+
+ Ok(())
  }
 }
 
@@ -69,8 +103,10 @@ pub struct RuntimeInterface<'a> {
 }
 
 impl<'a> RuntimeInterface<'a> {
- pub fn latest_versions(&self) -> impl Iterator<Item = (u32, &TraitItemMethod)> {
- self.items.iter().map(|(_, item)| item.latest_version())
+ /// Returns an iterator over all runtime interface function
+ /// [`latest_version_to_call`](RuntimeInterfaceFunction::latest_version).
+ pub fn latest_versions_to_call(&self) -> impl Iterator<Item = (u32, &TraitItemMethod)> {
+ self.items.iter().filter_map(|(_, item)| item.latest_version_to_call())
  }
 
  pub fn all_versions(&self) -> impl Iterator<Item = (u32, &TraitItemMethod)> {
@@ -199,36 +235,55 @@ fn get_trait_methods<'a>(trait_def: &'a ItemTrait) -> impl Iterator<Item = &'a T
  })
 }
 
-/// Parse version attribute.
+/// The version attribute that can be found above a runtime interface function.
 ///
-/// Returns error if it is in incorrent format. Correct format is only `#[version(X)]`.
-fn parse_version_attribute(version: &Attribute) -> Result<u32> {
- let meta = version.parse_meta()?;
-
- let err = Err(Error::new(
- meta.span(),
- "Unexpected `version` attribute. The supported format is `#[version(1)]`",
- ));
-
- match meta {
- Meta::List(list) =>
- if list.nested.len() != 1 {
- err
- } else if let Some(NestedMeta::Lit(Lit::Int(i))) = list.nested.first() {
- i.base10_parse()
- } else {
- err
- },
- _ => err,
+/// Supports the following formats:
+/// - `#[version(1)]`
+/// - `#[version(1, register_only)]`
+///
+/// While this struct is only for parsing the inner parts inside the `()`.
+struct VersionAttribute {
+ version: u32,
+ register_only: Option<attributes::register_only>,
+}
+
+impl VersionAttribute {
+ /// Is this function version callable?
+ fn is_callable(&self) -> bool {
+ self.register_only.is_none()
+ }
+}
+
+impl Default for VersionAttribute {
+ fn default() -> Self {
+ Self { version: 1, register_only: None }
+ }
+}
+
+impl Parse for VersionAttribute {
+ fn parse(input: syn::parse::ParseStream) -> Result<Self> {
+ let version: LitInt = input.parse()?;
+ let register_only = if input.peek(token::Comma) {
+ let _ = input.parse::<token::Comma>();
+ Some(input.parse()?)
+ } else {
+ if !input.is_empty() {
+ return Err(Error::new(input.span(), "Unexpected token, expected `,`."))
+ }
+
+ None
+ };
+
+ Ok(Self { version: version.base10_parse()?, register_only })
  }
 }
 
-/// Return item version (`#[version(X)]`) attribute, if present.
-fn get_item_version(item: &TraitItemMethod) -> Result<Option<u32>> {
+/// Return [`VersionAttribute`], if present.
+fn get_item_version(item: &TraitItemMethod) -> Result<Option<VersionAttribute>> {
  item.attrs
  .iter()
  .find(|attr| attr.path.is_ident("version"))
- .map(|attr| parse_version_attribute(attr))
+ .map(|attr| attr.parse_args())
  .transpose()
 }
 
@@ -238,28 +293,18 @@ pub fn get_runtime_interface<'a>(trait_def: &'a ItemTrait) -> Result<RuntimeInte
 
  for item in get_trait_methods(trait_def) {
  let name = item.sig.ident.clone();
- let version = get_item_version(item)?.unwrap_or(1);
+ let version = get_item_version(item)?.unwrap_or_default();
+
+ if version.version < 1 {
+ return Err(Error::new(item.span(), "Version needs to be at least `1`."))
+ }
 
  match functions.entry(name.clone()) {
  Entry::Vacant(entry) => {
  entry.insert(RuntimeInterfaceFunction::new(version, item));
  },
  Entry::Occupied(mut entry) => {
- if let Some(existing_item) = entry.get().versions.get(&version) {
- let mut err = Error::new(item.span(), "Duplicated version attribute");
- err.combine(Error::new(
- existing_item.span(),
- "Previous version with the same number defined here",
- ));
-
- return Err(err)
- }
-
- let interface_item = entry.get_mut();
- if interface_item.latest_version < version {
- interface_item.latest_version = version;
- }
- interface_item.versions.insert(version, item);
+ entry.get_mut().add_version(version, item)?;
  },
  }
  }

primitives/runtime-interface/src/lib.rs

@@ -153,6 +153,22 @@ pub use sp_std;
 /// [17].to_vec()
 /// }
 ///
+/// /// Call function, different version and only being registered.
+/// ///
+/// /// This `register_only` version is only being registered, aka exposed to the runtime,
+/// /// but the runtime will still use the version 2 of this function. This is useful for when
+/// /// new host functions should be introduced. Adding new host functions requires that all
+/// /// nodes have the host functions available, because otherwise they fail at instantiation
+/// /// of the runtime. With `register_only` the function will not be used when compiling the
+/// /// runtime, but it will already be there for a future version of the runtime that will
+/// /// switch to using these host function.
+/// #[version(3, register_only)]
+/// fn call(data: &[u8]) -> Vec<u8> {
+/// // Here you could call some rather complex code that only compiles on native or
+/// // is way faster in native than executing it in wasm.
+/// [18].to_vec()
+/// }
+///
 /// /// A function can take a `&self` or `&mut self` argument to get access to the
 /// /// `Externalities`. (The generated method does not require
 /// /// this argument, so the function can be called just with the `optional` argument)
@@ -177,12 +193,14 @@ pub use sp_std;
 /// trait Interface {
 /// fn call_version_1(data: &[u8]) -> Vec<u8>;
 /// fn call_version_2(data: &[u8]) -> Vec<u8>;
+/// fn call_version_3(data: &[u8]) -> Vec<u8>;
 /// fn set_or_clear_version_1(&mut self, optional: Option<Vec<u8>>);
 /// }
 ///
 /// impl Interface for &mut dyn sp_externalities::Externalities {
 /// fn call_version_1(data: &[u8]) -> Vec<u8> { Vec::new() }
 /// fn call_version_2(data: &[u8]) -> Vec<u8> { [17].to_vec() }
+/// fn call_version_3(data: &[u8]) -> Vec<u8> { [18].to_vec() }
 /// fn set_or_clear_version_1(&mut self, optional: Option<Vec<u8>>) {
 /// match optional {
 /// Some(value) => self.set_storage([1, 2, 3, 4].to_vec(), value),
@@ -204,6 +222,10 @@ pub use sp_std;
 /// <&mut dyn sp_externalities::Externalities as Interface>::call_version_2(data)
 /// }
 ///
+/// fn call_version_3(data: &[u8]) -> Vec<u8> {
+/// <&mut dyn sp_externalities::Externalities as Interface>::call_version_3(data)
+/// }
+///
 /// pub fn set_or_clear(optional: Option<Vec<u8>>) {
 /// set_or_clear_version_1(optional)
 /// }
@@ -285,8 +307,8 @@ pub use sp_std;
 /// This instructs the macro to make two significant changes to the generated code:
 ///
 /// 1. The generated functions are not callable from the native side.
-/// 2. The trait as shown above is not implemented for `Externalities` and is instead
-/// implemented  for `FunctionExecutor` (from `sp-wasm-interface`).
+/// 2. The trait as shown above is not implemented for [`Externalities`] and is instead
+/// implemented for `FunctionExecutor` (from `sp-wasm-interface`).
 ///
 /// # Disable tracing
 /// By addding `no_tracing` to the list of options you can prevent the wasm-side interface from

primitives/runtime-interface/test-wasm/src/lib.rs

@@ -123,6 +123,15 @@ pub trait TestApi {
  data == 42
  }
 
+ fn test_versionning_register_only(&self, data: u32) -> bool {
+ data == 80
+ }
+
+ #[version(2, register_only)]
+ fn test_versionning_register_only(&self, data: u32) -> bool {
+ data == 42
+ }
+
  /// Returns the input values as tuple.
  fn return_input_as_tuple(
  a: Vec<u8>,
@@ -271,6 +280,13 @@ wasm_export_functions! {
  assert!(!test_api::test_versionning(102));
  }
 
+ fn test_versionning_register_only_works() {
+ // Ensure that we will import the version of the runtime interface function that
+ // isn't tagged with `register_only`.
+ assert!(!test_api::test_versionning_register_only(42));
+ assert!(test_api::test_versionning_register_only(80));
+ }
+
  fn test_return_input_as_tuple() {
  let a = vec![1, 3, 4, 5];
  let b = 10000;

primitives/runtime-interface/test/src/lib.rs

@@ -169,6 +169,11 @@ fn test_versionining_with_new_host_works() {
  call_wasm_method::<HostFunctions>(wasm_binary_deprecated_unwrap(), "test_versionning_works");
 }
 
+#[test]
+fn test_versionining_register_only() {
+ call_wasm_method::<HostFunctions>(wasm_binary_unwrap(), "test_versionning_register_only_works");
+}
+
 #[test]
 fn test_tracing() {
  use std::fmt;